From 6b86175e0f0f2e4c3348930250a7c5c78c34bbf2 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Fri, 3 Jan 2025 20:02:31 +0800 Subject: [PATCH 01/16] add the implementation of CODDTest for SQLite3 --- src/sqlancer/common/oracle/CODDTestBase.java | 25 + .../common/schema/AbstractTables.java | 24 + .../sqlite3/SQLite3ExpectedValueVisitor.java | 46 + src/sqlancer/sqlite3/SQLite3Options.java | 3 + .../sqlite3/SQLite3OracleFactory.java | 12 + .../sqlite3/SQLite3ToStringVisitor.java | 158 ++- src/sqlancer/sqlite3/SQLite3Visitor.java | 36 + .../sqlite3/ast/SQLite3Expression.java | 201 +++- src/sqlancer/sqlite3/ast/SQLite3Select.java | 32 + .../sqlite3/oracle/SQLite3CODDTestOracle.java | 1002 +++++++++++++++++ .../sqlite3/schema/SQLite3DataType.java | 17 + 11 files changed, 1552 insertions(+), 4 deletions(-) create mode 100644 src/sqlancer/common/oracle/CODDTestBase.java create mode 100644 src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java diff --git a/src/sqlancer/common/oracle/CODDTestBase.java b/src/sqlancer/common/oracle/CODDTestBase.java new file mode 100644 index 000000000..162a8eaed --- /dev/null +++ b/src/sqlancer/common/oracle/CODDTestBase.java @@ -0,0 +1,25 @@ +package sqlancer.common.oracle; + +import sqlancer.Main.StateLogger; +import sqlancer.MainOptions; +import sqlancer.SQLConnection; +import sqlancer.SQLGlobalState; +import sqlancer.common.query.ExpectedErrors; + +public abstract class CODDTestBase> implements TestOracle { + protected final S state; + protected final ExpectedErrors errors = new ExpectedErrors(); + protected final StateLogger logger; + protected final MainOptions options; + protected final SQLConnection con; + protected String auxiliaryQueryString; + protected String foldedQueryString; + protected String originalQueryString; + + public CODDTestBase(S state) { + this.state = state; + this.con = state.getConnection(); + this.logger = state.getLogger(); + this.options = state.getOptions(); + } +} \ No newline at end of file diff --git a/src/sqlancer/common/schema/AbstractTables.java b/src/sqlancer/common/schema/AbstractTables.java index 2afff82ab..c75cb30e1 100644 --- a/src/sqlancer/common/schema/AbstractTables.java +++ b/src/sqlancer/common/schema/AbstractTables.java @@ -34,4 +34,28 @@ public String columnNamesAsString(Function function) { return getColumns().stream().map(function).collect(Collectors.joining(", ")); } + + public void addTable(T table) { + if (!this.tables.contains(table)) { + this.tables.add(table); + columns.addAll(table.getColumns()); + } + } + + public void removeTable(T table) { + if (this.tables.contains(table)) { + this.tables.remove(table); + for (C c : table.getColumns()) { + columns.remove(c); + } + } + } + + public Boolean isContained(T table) { + return this.tables.contains(table); + } + + public int getSize() { + return this.tables.size(); + } } diff --git a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java index 1c148ee86..40f8ae20a 100644 --- a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java @@ -14,14 +14,21 @@ import sqlancer.sqlite3.ast.SQLite3Expression.InOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Join; import sqlancer.sqlite3.ast.SQLite3Expression.MatchOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Alias; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Distinct; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Exist; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ExpressionBag; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableAndColumnRef; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClasure; import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Subquery; import sqlancer.sqlite3.ast.SQLite3Expression.TypeLiteral; @@ -303,4 +310,43 @@ public void visit(SQLite3SetClause set) { visit(set.getRight()); } + @Override + public void visit(SQLite3Alias alias) { + print(alias); + print(alias.getOrigonalExpression()); + print(alias.getAliasExpression()); + } + + @Override + public void visit(SQLite3WithClasure withClasure) { + print(withClasure); + print(withClasure.getLeft()); + print(withClasure.getRight()); + } + + @Override + public void visit(SQLite3TableAndColumnRef tableAndColumnRef) { + print(tableAndColumnRef); + } + + @Override + public void visit(SQLite3Values values) { + print(values); + } + + @Override + public void visit(SQLite3ExpressionBag expr) { + print(expr); + print(expr.getInnerExpr()); + } + + @Override + public void visit(SQLite3Typeof expr) { + print(expr); + print(expr.getInnerExpr()); + } + + @Override + public void visit(SQLite3ResultMap tableSummary) { + } } diff --git a/src/sqlancer/sqlite3/SQLite3Options.java b/src/sqlancer/sqlite3/SQLite3Options.java index ec90f3bca..47a80e832 100644 --- a/src/sqlancer/sqlite3/SQLite3Options.java +++ b/src/sqlancer/sqlite3/SQLite3Options.java @@ -79,6 +79,9 @@ public class SQLite3Options implements DBMSSpecificOptions @Parameter(names = { "--max-num-indexes" }, description = "The maximum number of indexes that can be created") public int maxNumIndexes = 20; + @Parameter(names = { "--coddtest-model" }, description = "Apply CODDTest on expression, subquery, or random") + public String coddTestModel = "random"; + @Override public List getTestOracleFactory() { return Arrays.asList(oracles); diff --git a/src/sqlancer/sqlite3/SQLite3OracleFactory.java b/src/sqlancer/sqlite3/SQLite3OracleFactory.java index b1a9a1265..2e2f9f0f7 100644 --- a/src/sqlancer/sqlite3/SQLite3OracleFactory.java +++ b/src/sqlancer/sqlite3/SQLite3OracleFactory.java @@ -11,6 +11,7 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; +import sqlancer.sqlite3.oracle.SQLite3CODDTestOracle; import sqlancer.sqlite3.oracle.SQLite3Fuzzer; import sqlancer.sqlite3.oracle.SQLite3PivotedQuerySynthesisOracle; import sqlancer.sqlite3.oracle.tlp.SQLite3TLPAggregateOracle; @@ -96,6 +97,17 @@ public TestOracle create(SQLite3GlobalState globalState) thr oracles.add(AGGREGATE.create(globalState)); return new CompositeTestOracle(oracles, globalState); } + }, + CODDTest { + @Override + public TestOracle create(SQLite3GlobalState globalState) throws SQLException { + return new SQLite3CODDTestOracle(globalState); + } + + @Override + public boolean requiresAllTablesToContainRows() { + return true; + } }; } diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index b8b860943..45d9c8db2 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -1,7 +1,11 @@ package sqlancer.sqlite3; import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import sqlancer.IgnoreMeException; import sqlancer.Randomly; import sqlancer.common.visitor.ToStringVisitor; import sqlancer.sqlite3.ast.SQLite3Aggregate; @@ -11,6 +15,7 @@ import sqlancer.sqlite3.ast.SQLite3Case.SQLite3CaseWithoutBaseExpression; import sqlancer.sqlite3.ast.SQLite3Cast; import sqlancer.sqlite3.ast.SQLite3Constant; +import sqlancer.sqlite3.ast.SQLite3Constant.SQLite3NullConstant; import sqlancer.sqlite3.ast.SQLite3Expression; import sqlancer.sqlite3.ast.SQLite3Expression.BetweenOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Cast; @@ -19,12 +24,19 @@ import sqlancer.sqlite3.ast.SQLite3Expression.InOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Join; import sqlancer.sqlite3.ast.SQLite3Expression.MatchOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Alias; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Distinct; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Exist; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ExpressionBag; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableAndColumnRef; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClasure; import sqlancer.sqlite3.ast.SQLite3Expression.Subquery; import sqlancer.sqlite3.ast.SQLite3Expression.TypeLiteral; import sqlancer.sqlite3.ast.SQLite3Function; @@ -35,6 +47,7 @@ import sqlancer.sqlite3.ast.SQLite3WindowFunctionExpression; import sqlancer.sqlite3.ast.SQLite3WindowFunctionExpression.SQLite3WindowFunctionFrameSpecBetween; import sqlancer.sqlite3.ast.SQLite3WindowFunctionExpression.SQLite3WindowFunctionFrameSpecTerm; +import sqlancer.sqlite3.schema.SQLite3DataType; public class SQLite3ToStringVisitor extends ToStringVisitor implements SQLite3Visitor { @@ -99,6 +112,10 @@ public void visit(SQLite3Select s, boolean inner) { if (inner) { sb.append("("); } + if (s.getWithClasure() != null) { + visit(s.getWithClasure()); + sb.append(" "); + } sb.append("SELECT "); switch (s.getFromOptions()) { case DISTINCT: @@ -288,13 +305,20 @@ public void visit(InOperation op) { sb.append("("); visit(op.getLeft()); sb.append(" IN "); - sb.append("("); if (op.getRightExpressionList() != null) { + sb.append("("); visit(op.getRightExpressionList()); + sb.append(")"); } else { - visit(op.getRightSelect()); + if (op.getRightSelect() instanceof SQLite3Expression.SQLite3TableReference) { + visit(op.getRightSelect()); + } else { + sb.append("("); + visit(op.getRightSelect()); + sb.append(")"); + } } - sb.append(")"); + sb.append(")"); } @@ -305,6 +329,9 @@ public void visit(Subquery query) { @Override public void visit(SQLite3Exist exist) { + if (exist.getNegated()) { + sb.append(" NOT"); + } sb.append(" EXISTS "); if (exist.getExpression() instanceof SQLite3SetClause) { sb.append("("); @@ -482,4 +509,129 @@ public void visit(SQLite3SetClause set) { sb.append(SQLite3Visitor.asString(set.getRight())); } + @Override + public void visit(SQLite3Alias alias) { + sb.append("("); + visit(alias.getOrigonalExpression()); + sb.append(")"); + sb.append(" AS "); + visit(alias.getAliasExpression()); + } + + @Override + public void visit(SQLite3WithClasure withClasure) { + sb.append("WITH "); + visit(withClasure.getLeft()); + sb.append(" AS "); + visit(withClasure.getRight()); + } + + @Override + public void visit(SQLite3TableAndColumnRef tableAndColumnRef) { + sb.append(tableAndColumnRef.getString()); + } + + @Override + public void visit(SQLite3Values values) { + Map> vs = values.getValues(); + int size = vs.get(vs.keySet().iterator().next()).size(); + List columnNames = values.getColumns().stream().map(c->c.getName()).collect(Collectors.toList()); + sb.append("(VALUES "); + for (int i = 0; i < size; i++) { + sb.append("("); + for (String name : columnNames) { + if (vs.get(name).get(i).getDataType() == SQLite3DataType.NULL) { + visit(vs.get(name).get(i)); + } else { + sb.append("(CAST("); + visit(vs.get(name).get(i)); + sb.append(" AS "); + switch(vs.get(name).get(i).getDataType()) { + case BINARY: + sb.append("BLOB))"); + break; + case INT: + sb.append("INT))"); + break; + case TEXT: + sb.append("TEXT))"); + break; + case REAL: + sb.append("REAL))"); + break; + default: + throw new IgnoreMeException(); + } + } + + sb.append(", "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append("), "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append(")"); + } + + @Override + public void visit(SQLite3ExpressionBag expr) { + visit(expr.getInnerExpr()); + } + + @Override + public void visit(SQLite3Typeof expr) { + sb.append("typeof("); + visit(expr.getInnerExpr()); + sb.append(")"); + } + + @Override + public void visit(SQLite3ResultMap tableSummary) { + // we utlize CASE WHEN THEN END here + SQLite3Values values = tableSummary.getValues(); + List columnRefs = tableSummary.getColumns(); + List summary = tableSummary.getSummary(); + + Map> vs = values.getValues(); + int size = vs.get(vs.keySet().iterator().next()).size(); + if (size == 0) { + sb.append("("); + for (int j = 0; j < columnRefs.size(); ++j) { + visit(columnRefs.get(j)); + sb.append(" IS NULL AND "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append(")"); + return; + } + List columnNames = values.getColumns().stream().map(c->c.getName()).collect(Collectors.toList()); + sb.append(" CASE "); + for (int i = 0; i < size; i++) { + sb.append("WHEN "); + for (int j = 0; j < columnNames.size(); ++j) { + visit(columnRefs.get(j)); + if (vs.get(columnNames.get(j)).get(i) instanceof SQLite3NullConstant) { + sb.append(" IS NULL"); + } else { + sb.append(" = "); + sb.append(vs.get(columnNames.get(j)).get(i).toString()); + } + sb.append(" AND "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append("THEN "); + visit(summary.get(i)); + sb.append(" "); + } + sb.append("END "); + } } diff --git a/src/sqlancer/sqlite3/SQLite3Visitor.java b/src/sqlancer/sqlite3/SQLite3Visitor.java index f891c17f0..d5ebce096 100644 --- a/src/sqlancer/sqlite3/SQLite3Visitor.java +++ b/src/sqlancer/sqlite3/SQLite3Visitor.java @@ -13,14 +13,21 @@ import sqlancer.sqlite3.ast.SQLite3Expression.InOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Join; import sqlancer.sqlite3.ast.SQLite3Expression.MatchOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Alias; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Distinct; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Exist; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ExpressionBag; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixUnaryOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableAndColumnRef; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClasure; import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Subquery; import sqlancer.sqlite3.ast.SQLite3Expression.TypeLiteral; @@ -130,6 +137,21 @@ default void visit(SQLite3PostfixUnaryOperation exp) { void visit(SQLite3WindowFunctionFrameSpecBetween between); + + void visit(SQLite3Alias alias); + + void visit(SQLite3WithClasure withClasure); + + void visit(SQLite3TableAndColumnRef tableAndColumnRef); + + void visit(SQLite3Values values); + + void visit(SQLite3ExpressionBag expr); + + void visit(SQLite3Typeof expr); + + void visit(SQLite3ResultMap tableSummary); + default void visit(SQLite3Expression expr) { if (expr instanceof Sqlite3BinaryOperation) { visit((Sqlite3BinaryOperation) expr); @@ -193,6 +215,20 @@ default void visit(SQLite3Expression expr) { visit((SQLite3TableReference) expr); } else if (expr instanceof SQLite3SetClause) { visit((SQLite3SetClause) expr); + } else if (expr instanceof SQLite3Alias) { + visit((SQLite3Alias) expr); + } else if (expr instanceof SQLite3WithClasure) { + visit((SQLite3WithClasure) expr); + } else if (expr instanceof SQLite3TableAndColumnRef) { + visit((SQLite3TableAndColumnRef) expr); + } else if (expr instanceof SQLite3Values) { + visit((SQLite3Values) expr); + } else if (expr instanceof SQLite3ExpressionBag) { + visit((SQLite3ExpressionBag) expr); + } else if (expr instanceof SQLite3Typeof) { + visit((SQLite3Typeof) expr); + } else if (expr instanceof SQLite3ResultMap) { + visit((SQLite3ResultMap) expr); } else { throw new AssertionError(expr); } diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index 7b131b6b5..a14432eec 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -2,6 +2,7 @@ import java.util.List; import java.util.Optional; +import java.util.Map; import sqlancer.IgnoreMeException; import sqlancer.LikeImplementationHelper; @@ -113,9 +114,19 @@ public SQLite3CollateSequence getImplicitCollateSequence() { public static class SQLite3Exist extends SQLite3Expression { private final SQLite3Expression select; + private boolean negated = false; - public SQLite3Exist(SQLite3Expression select) { + public SQLite3Exist(SQLite3Expression select, boolean negated) { this.select = select; + this.negated = negated; + } + + public void setNegated(boolean negated) { + this.negated = negated; + } + + public boolean getNegated() { + return this.negated; } public SQLite3Expression getExpression() { @@ -1552,4 +1563,192 @@ public boolean omitBracketsWhenPrinting() { } } + public static class SQLite3WithClasure extends SQLite3Expression { + + private SQLite3Expression left; + private SQLite3Expression right; + + public SQLite3WithClasure(SQLite3Expression left, SQLite3Expression right) { + this.left = left; + this.right = right; + } + + public SQLite3Expression getLeft() { + return this.left; + } + + public SQLite3Expression getRight() { + return this.right; + } + + public void updateRight(SQLite3Expression right) { + this.right = right; + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + } + + public static class SQLite3Alias extends SQLite3Expression { + + private SQLite3Expression origonalExpression; + private SQLite3Expression aliasExpression; + + public SQLite3Alias(SQLite3Expression origonalExpression, SQLite3Expression aliasExpression) { + this.origonalExpression = origonalExpression; + this.aliasExpression = aliasExpression; + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + + public SQLite3Expression getOrigonalExpression() { + return origonalExpression; + } + + public SQLite3Expression getAliasExpression() { + return aliasExpression; + } + } + + public static class SQLite3TableAndColumnRef extends SQLite3Expression { + + private final SQLite3Table table; + + public SQLite3TableAndColumnRef(SQLite3Table table) { + this.table = table; + } + + public SQLite3Table getTable() { + return this.table; + } + + public String getString() { + StringBuilder sb = new StringBuilder(); + sb.append(table.getName()); + sb.append("("); + for (SQLite3Column c : this.table.getColumns()) { + sb.append(c.getName()); + sb.append(", "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append(")"); + return sb.toString(); + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + } + + public static class SQLite3Values extends SQLite3Expression { + + private final Map> values; + private final List columns; + + public SQLite3Values(Map> values, List columns) { + this.values = values; + this.columns = columns; + } + + public Map> getValues() { + return this.values; + } + + public List getColumns() { + return this.columns; + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + } + + public static class SQLite3ExpressionBag extends SQLite3Expression { + private SQLite3Expression innerExpr; + + public SQLite3ExpressionBag(SQLite3Expression innerExpr) { + this.innerExpr = innerExpr; + } + + public void updateInnerExpr(SQLite3Expression innerExpr) { + this.innerExpr = innerExpr; + } + + public SQLite3Expression getInnerExpr() { + return innerExpr; + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + + } + + public static class SQLite3Typeof extends SQLite3Expression { + private SQLite3Expression innerExpr; + + public SQLite3Typeof(SQLite3Expression innerExpr) { + this.innerExpr = innerExpr; + } + + public SQLite3Expression getInnerExpr() { + return innerExpr; + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + + } + + public static class SQLite3ResultMap extends SQLite3Expression { + private final SQLite3Values values; + private final List columns; + private final List summary; + private final SQLite3DataType summaryDataType; + + public SQLite3ResultMap(SQLite3Values values, List columns, List summary, SQLite3DataType summaryDataType) { + this.values = values; + this.columns = columns; + this.summary = summary; + this.summaryDataType = summaryDataType; + + Map> vs = values.getValues(); + if (vs.get(vs.keySet().iterator().next()).size() != summary.size()) { + throw new AssertionError(); + } + } + + public SQLite3Values getValues() { + return this.values; + } + + public List getColumns() { + return this.columns; + } + + public List getSummary() { + return this.summary; + } + + public SQLite3DataType getSummaryDataType() { + return this.summaryDataType; + } + + @Override + public SQLite3CollateSequence getExplicitCollateSequence() { + return null; + } + + } } diff --git a/src/sqlancer/sqlite3/ast/SQLite3Select.java b/src/sqlancer/sqlite3/ast/SQLite3Select.java index 57eb91b0b..042aab856 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Select.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Select.java @@ -4,6 +4,7 @@ import java.util.Collections; import java.util.List; +import sqlancer.IgnoreMeException; import sqlancer.common.ast.newast.Select; import sqlancer.sqlite3.SQLite3Visitor; import sqlancer.sqlite3.ast.SQLite3Expression.Join; @@ -24,6 +25,7 @@ public class SQLite3Select extends SQLite3Expression private List fetchColumns = Collections.emptyList(); private List joinStatements = Collections.emptyList(); private SQLite3Expression havingClause; + private SQLite3WithClasure withClause = null; public SQLite3Select() { } @@ -42,6 +44,7 @@ public SQLite3Select(SQLite3Select other) { joinStatements.add(new Join(j)); } havingClause = other.havingClause; + withClause = other.withClause; } public enum SelectType { @@ -161,4 +164,33 @@ public SQLite3Expression getHavingClause() { public String asString() { return SQLite3Visitor.asString(this); } + + public void setWithClasure(SQLite3WithClasure withClasure) { + this.withClause = withClasure; + } + + public void updateWithClasureRight(SQLite3Expression withClasureRight) { + this.withClause.updateRight(withClasureRight); + } + + public SQLite3Expression getWithClasure() { + return this.withClause; + } + + public void replaceFromTable(String tableName, SQLite3Expression newFromExpression) { + int replaceIdx = -1; + for (int i = 0; i < fromList.size(); ++i) { + SQLite3Expression f = fromList.get(i); + if (f instanceof SQLite3TableReference) { + SQLite3TableReference tableRef = (SQLite3TableReference) f; + if (tableRef.getTable().getName() == tableName) { + replaceIdx = i; + } + } + } + if (replaceIdx == -1) { + throw new IgnoreMeException(); + } + fromList.set(replaceIdx, newFromExpression); + } } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java new file mode 100644 index 000000000..d20e6e9be --- /dev/null +++ b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java @@ -0,0 +1,1002 @@ +package sqlancer.sqlite3.oracle; + +import java.math.BigDecimal; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Main; +import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.common.oracle.CODDTestBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.sqlite3.SQLite3Errors; +import sqlancer.sqlite3.SQLite3GlobalState; +import sqlancer.sqlite3.SQLite3Provider; +import sqlancer.sqlite3.SQLite3Visitor; +import sqlancer.sqlite3.ast.SQLite3Aggregate; +import sqlancer.sqlite3.ast.SQLite3Constant; +import sqlancer.sqlite3.ast.SQLite3Constant.SQLite3TextConstant; +import sqlancer.sqlite3.ast.SQLite3Expression; +import sqlancer.sqlite3.ast.SQLite3Expression.InOperation; +import sqlancer.sqlite3.ast.SQLite3Expression.Join; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Alias; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ColumnName; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Exist; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ExpressionBag; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3PostfixText; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableAndColumnRef; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3TableReference; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClasure; +import sqlancer.sqlite3.ast.SQLite3Expression.Join.JoinType; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm.Ordering; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; +import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation.BinaryOperator; +import sqlancer.sqlite3.ast.SQLite3Select; +import sqlancer.sqlite3.gen.SQLite3Common; +import sqlancer.sqlite3.gen.SQLite3ExpressionGenerator; +import sqlancer.sqlite3.schema.SQLite3DataType; +import sqlancer.sqlite3.schema.SQLite3Schema; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Column; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Table; +import sqlancer.sqlite3.schema.SQLite3Schema.SQLite3Tables; + +public class SQLite3CODDTestOracle extends CODDTestBase implements TestOracle { + + private final SQLite3Schema s; + private SQLite3ExpressionGenerator gen; + private Reproducer reproducer; + + private String tempTableName = "temp_table"; + + private SQLite3Expression foldedExpr; + private SQLite3Expression constantResOfFoldedExpr; + + private List tablesFromOuterContext = new ArrayList<>(); + private List joinsInExpr = null; + + Map> auxiliaryQueryResult = new HashMap<>(); + Map> selectResult = new HashMap<>(); + + Boolean useSubqueryAsFoldedExpr; + Boolean useCorrelatedSubqueryAsFoldedExpr; + + public SQLite3CODDTestOracle(SQLite3GlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + SQLite3Errors.addExpectedExpressionErrors(errors); + SQLite3Errors.addMatchQueryErrors(errors); + SQLite3Errors.addQueryErrors(errors); + errors.add("misuse of aggregate"); + errors.add("misuse of window function"); + errors.add("second argument to nth_value must be a positive integer"); + errors.add("no such table"); + errors.add("no query solution"); + errors.add("unable to use function MATCH in the requested context"); + errors.add("[SQLITE_ERROR] SQL error or missing database (unrecognized token:"); + } + + @Override + public void check() throws SQLException { + reproducer = null; + + useSubqueryAsFoldedExpr = useSubquery(); + useCorrelatedSubqueryAsFoldedExpr = useCorrelatedSubquery(); + + SQLite3Select auxiliaryQuery = null; + if (useSubqueryAsFoldedExpr) { + if (useCorrelatedSubqueryAsFoldedExpr) { + auxiliaryQuery = genSelectWithCorrelatedSubquery(null, null); + auxiliaryQueryString = SQLite3Visitor.asString(auxiliaryQuery); + + auxiliaryQueryResult.putAll(selectResult); + } else { + auxiliaryQuery = genSelectExpression(null, null); + auxiliaryQueryString = SQLite3Visitor.asString(auxiliaryQuery); + auxiliaryQueryResult = getQueryResult(auxiliaryQueryString, state); + } + } else { + auxiliaryQuery = genSimpleSelect(null, null); + auxiliaryQueryString = SQLite3Visitor.asString(auxiliaryQuery); + + auxiliaryQueryResult.putAll(selectResult); + } + + + SQLite3Select originalQuery = null; + + Map> foldedResult = new HashMap<>(); + Map> originalResult = new HashMap<>(); + + // dependent expression + if (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) { + // original query + SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(this.foldedExpr); + originalQuery = this.genSelectExpression(null, specificCondition); + originalQueryString = SQLite3Visitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + specificCondition.updateInnerExpr(this.constantResOfFoldedExpr); + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + // independent expression + // empty result, put the inner query in (NOT) EXIST + else if (auxiliaryQueryResult.size() == 0 || auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().iterator().next()).size() == 0) { + boolean isNegated = Randomly.getBoolean() ? false : true; + // original query + SQLite3Exist existExpr = new SQLite3Exist(new SQLite3Select(auxiliaryQuery), isNegated); + SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(existExpr); + + originalQuery = this.genSelectExpression(null, specificCondition); + originalQueryString = SQLite3Visitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + SQLite3Expression equivalentExpr = isNegated ? SQLite3Constant.createTrue() : SQLite3Constant.createFalse(); + specificCondition.updateInnerExpr(equivalentExpr); + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + // Scalar Subquery: 1 column and 1 row, consider the inner query as a constant + else if (auxiliaryQueryResult.size() == 1 && auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]).size() == 1 && Randomly.getBoolean()) { + // original query + SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(auxiliaryQuery); + originalQuery = this.genSelectExpression(null, specificCondition); + originalQueryString = SQLite3Visitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + SQLite3Expression equivalentExpr = auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]).get(0); + specificCondition.updateInnerExpr(equivalentExpr);; + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + // one column + else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowProbability()) { + // else if (auxiliaryQueryResult.size() == 1 && false) { + // original query + List columns = s.getRandomTableNonEmptyTables().getColumns(); + SQLite3ColumnName selectedColumn = new SQLite3ColumnName(Randomly.fromList(columns), null); + SQLite3Table selectedTable = selectedColumn.getColumn().getTable(); + InOperation INOperation = new InOperation(selectedColumn, new SQLite3Select(auxiliaryQuery)); + SQLite3ExpressionBag specificCondition = new SQLite3ExpressionBag(INOperation); + + originalQuery = this.genSelectExpression(selectedTable, specificCondition); + originalQueryString = SQLite3Visitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + // folded query + // can not use IN VALUES here, because there is no affinity for the right operand of IN when right operand is a list + try { + SQLite3Table t = this.createTemporaryTable(auxiliaryQuery, "intable"); + SQLite3TableReference equivalentTable = new SQLite3TableReference(t); + INOperation = new InOperation(selectedColumn, equivalentTable); + specificCondition.updateInnerExpr(INOperation); + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } finally { + dropTemporaryTable("intable"); + } + } + // There is not `ANY` and `ALL` operator in SQLite3 + // Row Subquery + else { + // original query + SQLite3Table temporaryTable = this.genTemporaryTable(auxiliaryQuery, this.tempTableName); + originalQuery = this.genSelectExpression(temporaryTable, null); + SQLite3TableAndColumnRef tableAndColumnRef = new SQLite3TableAndColumnRef(temporaryTable); + SQLite3WithClasure withClasure = new SQLite3WithClasure(tableAndColumnRef, new SQLite3Select(auxiliaryQuery)); + originalQuery.setWithClasure(withClasure); + originalQueryString = SQLite3Visitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + // folded query + if (Randomly.getBoolean() && this.testCommonTableExpression()) { + // there are too many false positives + // common table expression + // folded query: WITH table AS VALUES () + SQLite3Values values = new SQLite3Values(auxiliaryQueryResult, temporaryTable.getColumns()); + originalQuery.updateWithClasureRight(values); + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } else if (Randomly.getBoolean() && this.testDerivedTable()) { + // derived table + // folded query: SELECT FROM () AS table + originalQuery.setWithClasure(null); + SQLite3TableReference tempTableRef = new SQLite3TableReference(temporaryTable); + SQLite3Alias alias = new SQLite3Alias(new SQLite3Select(auxiliaryQuery), tempTableRef); + originalQuery.replaceFromTable(this.tempTableName, alias); + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } else if (this.testInsert()){ + // there are too many false positives + // folded query: CREATE the table and INSERT INTO table subquery + try { + this.createTemporaryTable(auxiliaryQuery, this.tempTableName); + originalQuery.setWithClasure(null); + foldedQueryString = SQLite3Visitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } finally { + dropTemporaryTable(this.tempTableName); + } + } else { + throw new IgnoreMeException(); + } + } + if (foldedResult == null || originalResult == null) { + throw new IgnoreMeException(); + } + if (foldedQueryString.equals(originalQueryString)) { + throw new IgnoreMeException(); + } + if (!compareResult(foldedResult, originalResult)) { + reproducer = null; // TODO + state.getState().getLocalState().log(auxiliaryQueryString + ";\n" + foldedQueryString + ";\n" + originalQueryString + ";"); + throw new AssertionError(auxiliaryQueryResult.toString() + " " + foldedResult.toString() + " " + originalResult.toString()); + } + } + + private SQLite3Select genSelectExpression(SQLite3Table tempTable, SQLite3Expression specificCondition) { + SQLite3Tables randomTables = s.getRandomTableNonEmptyTables(); + if (tempTable != null) { + randomTables.addTable(tempTable); + } + if (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) { + for (SQLite3Table t : this.tablesFromOuterContext) { + randomTables.addTable(t); + } + if (this.joinsInExpr != null) { + for (Join j : this.joinsInExpr) { + SQLite3Table t = j.getTable(); + randomTables.removeTable(t); + } + } + } + + List columns = randomTables.getColumns(); + if ((!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) && this.joinsInExpr != null) { + for (Join j : this.joinsInExpr) { + SQLite3Table t = j.getTable(); + columns.addAll(t.getColumns()); + } + } + gen = new SQLite3ExpressionGenerator(state).setColumns(columns); + List tables = randomTables.getTables(); + List joinStatements = new ArrayList<>(); + if ((!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) && this.joinsInExpr != null) { + joinStatements.addAll(this.joinsInExpr); + this.joinsInExpr = null; + } + else if (Randomly.getBoolean()) { + joinStatements = genJoinExpression(gen, tables, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null, false); + } + List tableRefs = SQLite3Common.getTableRefs(tables, s); + SQLite3Select select = new SQLite3Select(); + select.setFromList(tableRefs); + if (joinStatements.size() > 0) { + select.setJoinClauses(joinStatements); + } + + SQLite3Expression randomWhereCondition = gen.generateExpression(); + SQLite3Expression whereCondition = null; + if (specificCondition != null) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + whereCondition = new SQLite3Expression.Sqlite3BinaryOperation(randomWhereCondition, specificCondition, operator); + } else { + whereCondition = randomWhereCondition; + } + select.setWhereClause(whereCondition); + + if (Randomly.getBoolean()) { + select.setOrderByClauses(genOrderBysExpression(gen, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null)); + } + + if (Randomly.getBoolean()) { + List selectedColumns = Randomly.nonEmptySubset(columns); + List selectedAlias = new LinkedList<>(); + for (int i = 0; i < selectedColumns.size(); ++i) { + SQLite3ColumnName originalName = new SQLite3ColumnName(selectedColumns.get(i), null); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + String.valueOf(i)), null); + SQLite3Alias columnAlias = new SQLite3Alias(originalName, aliasName); + selectedAlias.add(columnAlias); + } + select.setFetchColumns(selectedAlias); + } else { + SQLite3ColumnName aggr = new SQLite3ColumnName(Randomly.fromList(columns), null); + SQLite3Provider.mustKnowResult = true; + SQLite3Expression originalName = new SQLite3Aggregate(Arrays.asList(aggr), SQLite3Aggregate.SQLite3AggregateFunction.getRandom()); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c0"), null); + SQLite3Alias columnAlias = new SQLite3Alias(originalName, aliasName); + select.setFetchColumns(Arrays.asList(columnAlias)); + if (Randomly.getBooleanWithRatherLowProbability()) { + List groupByClause = genGroupByClause(columns, specificCondition); + select.setGroupByClause(groupByClause); + if (groupByClause.size() > 0 && Randomly.getBooleanWithRatherLowProbability()) { + select.setHavingClause(genHavingClause(columns, specificCondition)); + } + } + } + return select; + } + + // For expression test + private SQLite3Select genSimpleSelect(SQLite3Table tempTable, SQLite3Expression specificCondition) { + SQLite3Tables randomTables = s.getRandomTableNonEmptyTables(); + if (tempTable != null) { + randomTables.addTable(tempTable); + } + if (!useSubqueryAsFoldedExpr) { + for (SQLite3Table t : this.tablesFromOuterContext) { + randomTables.addTable(t); + } + if (this.joinsInExpr != null) { + for (Join j : this.joinsInExpr) { + SQLite3Table t = j.getTable(); + randomTables.removeTable(t); + } + } + } + + List columns = randomTables.getColumns(); + if (!useSubqueryAsFoldedExpr && this.joinsInExpr != null) { + for (Join j : this.joinsInExpr) { + SQLite3Table t = j.getTable(); + columns.addAll(t.getColumns()); + } + } + gen = new SQLite3ExpressionGenerator(state).setColumns(columns); + List tables = randomTables.getTables(); + tablesFromOuterContext = randomTables.getTables(); + + if (joinsInExpr == null) { + if (Randomly.getBooleanWithRatherLowProbability()) { + joinsInExpr = genJoinExpression(gen, tables, null, true); + } else { + joinsInExpr = new ArrayList(); + } + } + + List tableRefs = SQLite3Common.getTableRefs(tables, s); + SQLite3Select select = new SQLite3Select(); + select.setFromList(tableRefs); + if (joinsInExpr != null && joinsInExpr.size() > 0) { + select.setJoinClauses(joinsInExpr); + } + + SQLite3Expression whereCondition = gen.generateExpression(); + if (specificCondition != null) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + whereCondition = new SQLite3Expression.Sqlite3BinaryOperation(whereCondition, specificCondition, operator); + } + this.foldedExpr = whereCondition; + + List fetchColumns = new ArrayList<>(); + int columnIdx = 0; + for (SQLite3Column c : randomTables.getColumns()) { + SQLite3ColumnName cRef = new SQLite3ColumnName(c, null); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + String.valueOf(columnIdx)), null); + SQLite3Alias columnAlias = new SQLite3Alias(cRef, aliasName); + fetchColumns.add(columnAlias); + columnIdx++; + } + + // add the expression to fetch clause + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + String.valueOf(columnIdx)), null); + SQLite3Alias columnAlias = new SQLite3Alias(whereCondition, aliasName); + fetchColumns.add(columnAlias); + + select.setFetchColumns(fetchColumns); + + originalQueryString = SQLite3Visitor.asString(select); + + Map> queryRes = null; + try { + queryRes = getQueryResult(originalQueryString, state); + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(e.getMessage()); + } + } + if (queryRes.get("c0").size() == 0) { + throw new IgnoreMeException(); + } + + // save the result first + selectResult.clear(); + selectResult.putAll(queryRes); + + // get the summary from results + List summary = queryRes.remove("c" + String.valueOf(columnIdx)); + + List tempColumnList = new ArrayList<>(); + + for (int i = 0; i < fetchColumns.size() - 1; ++i) { + // do not put the last fetch column to values + SQLite3Alias cAlias = (SQLite3Alias) fetchColumns.get(i); + SQLite3ColumnName cRef = (SQLite3ColumnName) cAlias.getOrigonalExpression(); + SQLite3Column column = cRef.getColumn(); + String columnName = SQLite3Visitor.asString(cAlias.getAliasExpression()); + SQLite3Column newColumn = new SQLite3Column(columnName, column.getType(), false, false, null); + tempColumnList.add(newColumn); + } + List columnRef = new ArrayList<>(); + for (SQLite3Column c : randomTables.getColumns()) { + columnRef.add(new SQLite3ColumnName(c, null)); + } + if (tempColumnList.size() != queryRes.size()) { + throw new AssertionError(); + } + SQLite3Values values = new SQLite3Values(queryRes, tempColumnList); + this.constantResOfFoldedExpr = new SQLite3ResultMap(values, columnRef, summary, null); + + return select; + } + + private SQLite3Select genSelectWithCorrelatedSubquery(SQLite3Table selectedTable, SQLite3Expression specificCondition) { + // do not support join now + this.joinsInExpr = new ArrayList(); + + SQLite3Tables outerQueryRandomTables = s.getRandomTableNonEmptyTables(); + SQLite3Tables innerQueryRandomTables = s.getRandomTableNonEmptyTables(); + + if (selectedTable != null) { + innerQueryRandomTables.addTable(selectedTable); + } + + List innerQueryFromTables = new ArrayList<>(); + for (SQLite3Table t : innerQueryRandomTables.getTables()) { + if (!outerQueryRandomTables.isContained(t)) { + innerQueryFromTables.add(new SQLite3TableReference(t)); + } + } + for (SQLite3Table t : outerQueryRandomTables.getTables()) { + if (innerQueryRandomTables.isContained(t)) { + innerQueryRandomTables.removeTable(t); + + List newColumns = new ArrayList<>(); + for (SQLite3Column c : t.getColumns()) { + SQLite3Column newColumn = new SQLite3Column(c.getName(), c.getType(), false, null, false); + newColumns.add(newColumn); + } + SQLite3Table newTable = new SQLite3Table(t.getName() + "a", newColumns, null, true, false, false, false); + for (SQLite3Column c : newColumns) { + c.setTable(newTable); + } + innerQueryRandomTables.addTable(newTable); + + SQLite3Alias alias = new SQLite3Alias(new SQLite3TableReference(t), new SQLite3TableReference(newTable)); + innerQueryFromTables.add(alias); + } + } + + List innerQueryColumns = new ArrayList<>(); + innerQueryColumns.addAll(innerQueryRandomTables.getColumns()); + innerQueryColumns.addAll(outerQueryRandomTables.getColumns()); + gen = new SQLite3ExpressionGenerator(state).setColumns(innerQueryColumns); + + SQLite3Select innerQuery = new SQLite3Select(); + innerQuery.setFromList(innerQueryFromTables); + + SQLite3Expression innerQueryWhereCondition = gen.generateExpression(); + if (specificCondition != null) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + innerQueryWhereCondition = new SQLite3Expression.Sqlite3BinaryOperation(innerQueryWhereCondition, specificCondition, operator); + } + innerQuery.setWhereClause(innerQueryWhereCondition); + + // use aggregate function in fetch column + SQLite3ColumnName innerQueryAggr = new SQLite3ColumnName(Randomly.fromList(innerQueryRandomTables.getColumns()), null); + SQLite3Provider.mustKnowResult = true; + SQLite3Expression innerQueryAggrName = new SQLite3Aggregate(Arrays.asList(innerQueryAggr), SQLite3Aggregate.SQLite3AggregateFunction.getRandom()); + innerQuery.setFetchColumns(Arrays.asList(innerQueryAggrName)); + if (Randomly.getBooleanWithRatherLowProbability()) { + List groupByClause = genGroupByClause(innerQueryColumns, specificCondition); + innerQuery.setGroupByClause(groupByClause); + if (groupByClause.size() > 0 && Randomly.getBooleanWithRatherLowProbability()) { + innerQuery.setHavingClause(genHavingClause(innerQueryColumns, specificCondition)); + } + } + + this.foldedExpr = innerQuery; + + + // outer query + SQLite3Select outerQuery = new SQLite3Select(); + outerQuery.setFromList(SQLite3Common.getTableRefs(outerQueryRandomTables.getTables(), s)); + tablesFromOuterContext = outerQueryRandomTables.getTables(); + + List outerQueryFetchColumns = new ArrayList<>(); + int columnIdx = 0; + for (SQLite3Column c : outerQueryRandomTables.getColumns()) { + SQLite3ColumnName cRef = new SQLite3ColumnName(c, null); + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + String.valueOf(columnIdx)), null); + SQLite3Alias columnAlias = new SQLite3Alias(cRef, aliasName); + outerQueryFetchColumns.add(columnAlias); + columnIdx++; + } + + // add the expression to fetch clause + SQLite3ColumnName aliasName = new SQLite3ColumnName(SQLite3Column.createDummy("c" + String.valueOf(columnIdx)), null); + SQLite3Alias columnAlias = new SQLite3Alias(innerQuery, aliasName); + outerQueryFetchColumns.add(columnAlias); + + outerQuery.setFetchColumns(outerQueryFetchColumns); + + originalQueryString = SQLite3Visitor.asString(outerQuery); + + Map> queryRes = null; + try { + queryRes = getQueryResult(originalQueryString, state); + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(e.getMessage()); + } + } + if (queryRes.get("c0").size() == 0) { + throw new IgnoreMeException(); + } + + // save the result first + selectResult.clear(); + selectResult.putAll(queryRes); + + // get the summary from results + List summary = queryRes.remove("c" + String.valueOf(columnIdx)); + + List tempColumnList = new ArrayList<>(); + + for (int i = 0; i < outerQueryFetchColumns.size() - 1; ++i) { + // do not put the last fetch column to values + SQLite3Alias cAlias = (SQLite3Alias) outerQueryFetchColumns.get(i); + SQLite3ColumnName cRef = (SQLite3ColumnName) cAlias.getOrigonalExpression(); + SQLite3Column column = cRef.getColumn(); + String columnName = SQLite3Visitor.asString(cAlias.getAliasExpression()); + SQLite3Column newColumn = new SQLite3Column(columnName, column.getType(), false, false, null); + tempColumnList.add(newColumn); + } + List columnRef = new ArrayList<>(); + for (SQLite3Column c : outerQueryRandomTables.getColumns()) { + columnRef.add(new SQLite3ColumnName(c, null)); + } + if (tempColumnList.size() != queryRes.size()) { + throw new AssertionError(); + } + SQLite3Values values = new SQLite3Values(queryRes, tempColumnList); + this.constantResOfFoldedExpr = new SQLite3ResultMap(values, columnRef, summary, null); + + return outerQuery; + } + + private List genJoinExpression(SQLite3ExpressionGenerator gen, List tables, SQLite3Expression specificCondition, boolean joinForExperssion) { + List joinStatements = new ArrayList<>(); + if (!state.getDbmsSpecificOptions().testJoins) { + return joinStatements; + } + List options = new ArrayList<>(Arrays.asList(JoinType.values())); + if (Randomly.getBoolean() && tables.size() > 1) { + int nrJoinClauses = (int) Randomly.getNotCachedInteger(0, tables.size()); + // Natural join is incompatible with other joins + // because it needs unique column names + // while other joins will produce duplicate column names + if (nrJoinClauses > 1 || joinForExperssion) { + options.remove(JoinType.NATURAL); + } + for (int i = 0; i < nrJoinClauses; i++) { + SQLite3Expression randomOnCondition = gen.generateExpression(); + SQLite3Expression onCondition = null; + if (specificCondition != null && Randomly.getBooleanWithRatherLowProbability()) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + onCondition = new SQLite3Expression.Sqlite3BinaryOperation(randomOnCondition, specificCondition, operator); + } else { + onCondition = randomOnCondition; + } + + SQLite3Table table = Randomly.fromList(tables); + tables.remove(table); + JoinType selectedOption = Randomly.fromList(options); + if (selectedOption == JoinType.NATURAL) { + // NATURAL joins do not have an ON clause + onCondition = null; + } + Join j = new SQLite3Expression.Join(table, onCondition, selectedOption); + joinStatements.add(j); + } + + } + return joinStatements; + } + + private List genOrderBysExpression(SQLite3ExpressionGenerator gen, SQLite3Expression specificCondition) { + List expressions = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber() + 1; i++) { + expressions.add(genOrderingTerm(gen, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null)); + } + return expressions; + } + + private SQLite3Expression genOrderingTerm(SQLite3ExpressionGenerator gen, SQLite3Expression specificCondition) { + SQLite3Expression expr = gen.generateExpression(); + if (specificCondition != null && Randomly.getBooleanWithRatherLowProbability()) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + expr = new SQLite3Expression.Sqlite3BinaryOperation(expr, specificCondition, operator); + } + // COLLATE is potentially already generated + if (Randomly.getBoolean()) { + expr = new SQLite3OrderingTerm(expr, Ordering.getRandomValue()); + } + if (state.getDbmsSpecificOptions().testNullsFirstLast && Randomly.getBoolean()) { + expr = new SQLite3PostfixText(expr, Randomly.fromOptions(" NULLS FIRST", " NULLS LAST"), + null /* expr.getExpectedValue() */) { + @Override + public boolean omitBracketsWhenPrinting() { + return true; + } + }; + } + return expr; + } + + private List genGroupByClause(List columns, SQLite3Expression specificCondition) { + errors.add("GROUP BY term out of range"); + if (Randomly.getBoolean()) { + List collect = new ArrayList<>(); + for (int i = 0; i < Randomly.smallNumber(); i++) { + SQLite3Expression expr = new SQLite3ExpressionGenerator(state).setColumns(columns).generateExpression(); + if (specificCondition != null && Randomly.getBooleanWithRatherLowProbability()) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + expr = new SQLite3Expression.Sqlite3BinaryOperation(expr, specificCondition, operator); + } + collect.add(expr); + } + return collect; + } + return Collections.emptyList(); + } + + private SQLite3Expression genHavingClause(List columns, SQLite3Expression specificCondition) { + SQLite3Expression expr = new SQLite3ExpressionGenerator(state).setColumns(columns).generateExpression(); + if (specificCondition != null && Randomly.getBooleanWithRatherLowProbability()) { + BinaryOperator operator = BinaryOperator.getRandomOperator(); + expr = new SQLite3Expression.Sqlite3BinaryOperation(expr, specificCondition, operator); + } + return expr; + } + + private Map> getQueryResult(String queryString, SQLite3GlobalState state) throws SQLException { + Map> result = new LinkedHashMap<>(); + if (options.logEachSelect()) { + logger.writeCurrentNoLineBreak(queryString); + } + Statement stmt = null; + try { + stmt = this.con.createStatement(); + stmt.setQueryTimeout(600); + ResultSet rs = null; + try { + rs = stmt.executeQuery(queryString); + ResultSetMetaData metaData = rs.getMetaData(); + Integer columnCount = metaData.getColumnCount(); + Map idxNameMap = new HashMap<>(); + for (int i = 1; i <= columnCount; i++) { + result.put("c" + String.valueOf(i-1), new ArrayList<>()); + idxNameMap.put(i, "c" + String.valueOf(i-1)); + } + + int resultRows = 0; + while (rs.next()) { + for (int i = 1; i <= columnCount; i++) { + try { + Object value = rs.getObject(i); + SQLite3Constant constant; + if (rs.wasNull()) { + constant = SQLite3Constant.createNullConstant(); + } + + else if (value instanceof Integer) { + constant = SQLite3Constant.createIntConstant(Long.valueOf((Integer) value)); + } else if (value instanceof Short) { + constant = SQLite3Constant.createIntConstant(Long.valueOf((Short) value)); + } else if (value instanceof Long) { + constant = SQLite3Constant.createIntConstant((Long) value); + } + + else if (value instanceof Double) { + constant = SQLite3Constant.createRealConstant((double) value); + } else if (value instanceof Float) { + constant = SQLite3Constant.createRealConstant(((Float) value).doubleValue()); + } else if (value instanceof BigDecimal) { + constant = SQLite3Constant.createRealConstant(((BigDecimal) value).doubleValue()); + } + + else if (value instanceof Byte) { + constant = SQLite3Constant.createBinaryConstant((byte[]) value); + } else if (value instanceof byte[]) { + constant = SQLite3Constant.createBinaryConstant((byte[]) value); + } else if (value instanceof Boolean) { + constant = SQLite3Constant.createBoolean((boolean) value); + } else if (value instanceof String) { + constant = SQLite3Constant.createTextConstant((String) value); + } else if (value == null) { + constant = SQLite3Constant.createNullConstant(); + } else { + throw new IgnoreMeException(); + } + List v = result.get(idxNameMap.get(i)); + v.add(constant); + } catch (SQLException e) { + System.out.println(e.getMessage()); + throw new IgnoreMeException(); + } + } + ++resultRows; + if (resultRows > 100) { + throw new IgnoreMeException(); + } + } + Main.nrSuccessfulActions.addAndGet(1); + rs.close(); + } catch (SQLException e) { + Main.nrUnsuccessfulActions.addAndGet(1); + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + state.getState().getLocalState().log(queryString); + throw new AssertionError(e.getMessage()); + } + } finally { + if (rs != null) { + rs.close(); + } + } + } finally { + if (stmt != null) { + stmt.close(); + } + } + return result; + } + + private SQLite3Table genTemporaryTable(SQLite3Select select, String tableName) { + List fetchColumns = select.getFetchColumns(); + int columnNumber = fetchColumns.size(); + Map idxTypeMap = getColumnTypeFromSelect(select); + + List databaseColumns = new ArrayList<>(); + for (int i = 0; i < columnNumber; ++i) { + String columnName = "c" + String.valueOf(i); + SQLite3Column column = new SQLite3Column(columnName, idxTypeMap.get(i), false, false, null); + databaseColumns.add(column); + } + SQLite3Table table = new SQLite3Table(tableName, databaseColumns, null, false, false, false, false); + for (SQLite3Column c : databaseColumns) { + c.setTable(table); + } + + return table; + } + + private SQLite3Table createTemporaryTable(SQLite3Select select, String tableName) throws SQLException { + String selectString = SQLite3Visitor.asString(select); + Map idxTypeMap = getColumnTypeFromSelect(select); + + Integer columnNumber = idxTypeMap.size(); + StringBuilder sb = new StringBuilder(); + sb.append("CREATE TABLE " + tableName + " ("); + for (int i = 0; i < columnNumber; ++i) { + String columnTypeName = ""; + if (idxTypeMap.get(i) != null) { + switch (idxTypeMap.get(i)) { + case INT: + case TEXT: + case REAL: + columnTypeName = idxTypeMap.get(i).name(); + break; + case BINARY: + columnTypeName = ""; + break; + default: + columnTypeName = ""; + } + } + sb.append("c" + String.valueOf(i) + " " + columnTypeName + ", "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append(");"); + String crateTableString = sb.toString(); + if (options.logEachSelect()) { + logger.writeCurrent(crateTableString); + } + Statement stmt = null; + try { + stmt = this.con.createStatement(); + try { + stmt.execute(crateTableString); + Main.nrSuccessfulActions.addAndGet(1); + } catch (SQLException e) { + Main.nrUnsuccessfulActions.addAndGet(1); + throw new IgnoreMeException(); + } + } finally { + if (stmt != null) { + stmt.close(); + } + } + + StringBuilder sb2 = new StringBuilder(); + sb2.append("INSERT INTO " + tableName + " "+ selectString); + String insertValueString = sb2.toString(); + if (options.logEachSelect()) { + logger.writeCurrent(insertValueString); + } + stmt = null; + try { + stmt = this.con.createStatement(); + try { + Main.nrSuccessfulActions.addAndGet(1); + stmt.execute(insertValueString); + } catch (SQLException e) { + Main.nrUnsuccessfulActions.addAndGet(1); + throw new IgnoreMeException(); + } + } finally { + if (stmt != null) { + stmt.close(); + } + } + + List databaseColumns = new ArrayList<>(); + for (int i = 0; i < columnNumber; ++i) { + String columnName = "c" + String.valueOf(i); + SQLite3Column column = new SQLite3Column(columnName, idxTypeMap.get(i), false, false, null); + databaseColumns.add(column); + } + SQLite3Table table = new SQLite3Table(tableName, databaseColumns, null, false, false, false, false); + for (SQLite3Column c : databaseColumns) { + c.setTable(table); + } + + return table; + } + + private void dropTemporaryTable(String tableName) throws SQLException { + String dropString = "DROP TABLE " + tableName + ";"; + if (options.logEachSelect()) { + logger.writeCurrent(dropString); + } + Statement stmt = null; + try { + stmt = this.con.createStatement(); + try { + stmt.execute(dropString); + Main.nrSuccessfulActions.addAndGet(1); + } catch (SQLException e) { + Main.nrUnsuccessfulActions.addAndGet(1); + throw new IgnoreMeException(); + } + } finally { + if (stmt != null) { + stmt.close(); + } + } + } + + private boolean compareResult(Map> r1, Map> r2) { + if (r1.size() != r2.size()) { + return false; + } + for (Map.Entry < String, List > entry: r1.entrySet()) { + String currentKey = entry.getKey(); + if (!r2.containsKey(currentKey)) { + return false; + } + List v1= entry.getValue(); + List v2= r2.get(currentKey); + if (v1.size() != v2.size()) { + return false; + } + List v1Value = new ArrayList<>(v1.stream().map(c -> c.toString()).collect(Collectors.toList())); + List v2Value = new ArrayList<>(v2.stream().map(c -> c.toString()).collect(Collectors.toList())); + Collections.sort(v1Value); + Collections.sort(v2Value); + if (!v1Value.equals(v2Value)) { + return false; + } + } + return true; + } + + private Map getColumnTypeFromSelect(SQLite3Select select) { + List fetchColumns = select.getFetchColumns(); + List newFetchColumns = new ArrayList<>(); + for(SQLite3Expression column : fetchColumns) { + newFetchColumns.add(column); + SQLite3Alias columnAlias = (SQLite3Alias) column; + SQLite3Expression typeofColumn = new SQLite3Typeof(columnAlias.getOrigonalExpression()); + newFetchColumns.add(typeofColumn); + } + SQLite3Select newSelect = new SQLite3Select(select); + newSelect.setFetchColumns(newFetchColumns); + Map> typeResult = null; + try { + typeResult = getQueryResult(SQLite3Visitor.asString(newSelect), state); + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(e.getMessage()); + } + } + + if (typeResult == null) { + throw new IgnoreMeException(); + } + Map idxTypeMap = new HashMap<>(); + for (int i = 0; i * 2 < typeResult.size(); ++i) { + String columnName = "c" + String.valueOf(i * 2 + 1); + SQLite3Expression t = typeResult.get(columnName).get(0); + SQLite3TextConstant tString = (SQLite3TextConstant) t; + String typeName = tString.asString(); + SQLite3DataType cType = SQLite3DataType.getTypeFromName(typeName); + idxTypeMap.put(i, cType); + } + + return idxTypeMap; + } + + public boolean useSubquery() { + if (this.state.getDbmsSpecificOptions().coddTestModel.equals("random")) { + return Randomly.getBoolean(); + } else if (this.state.getDbmsSpecificOptions().coddTestModel.equals("expression")) { + return false; + } else if (this.state.getDbmsSpecificOptions().coddTestModel.equals("subquery")) { + return true; + } else { + System.out.printf("Wrong option of --coddtest-model, should be one of: random, expression, subquery"); + System.exit(1); + return false; + } + } + + public boolean useCorrelatedSubquery() { + return Randomly.getBoolean(); + } + + public boolean testCommonTableExpression() { + return false; + } + public boolean testDerivedTable() { + return true; + } + public boolean testInsert() { + return false; + } + + @Override + public String getLastQueryString() { + return originalQueryString; + } + + @Override + public Reproducer getLastReproducer() { + return reproducer; + } +} diff --git a/src/sqlancer/sqlite3/schema/SQLite3DataType.java b/src/sqlancer/sqlite3/schema/SQLite3DataType.java index 87d91f451..8a343fed2 100644 --- a/src/sqlancer/sqlite3/schema/SQLite3DataType.java +++ b/src/sqlancer/sqlite3/schema/SQLite3DataType.java @@ -1,6 +1,23 @@ package sqlancer.sqlite3.schema; +import sqlancer.IgnoreMeException; + public enum SQLite3DataType { NULL, INT, TEXT, REAL, NONE, BINARY; + public static SQLite3DataType getTypeFromName(String name) { + if (name.equals("integer")) { + return INT; + } else if (name.equals("real")) { + return REAL; + } else if (name.equals("text")) { + return TEXT; + } else if (name.equals("blob")) { + return NONE; + } else if (name.equals("null")) { + return NULL; + } else { + throw new IgnoreMeException(); + } + } } From c6f2c4a29665e9d53648150eff11e7515913079f Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Sun, 5 Jan 2025 21:09:59 +0800 Subject: [PATCH 02/16] fix a typo and remove some redundant code of CODDTest for sqlite3 --- .../sqlite3/SQLite3ExpectedValueVisitor.java | 2 +- .../sqlite3/SQLite3ToStringVisitor.java | 2 +- .../sqlite3/ast/SQLite3Expression.java | 10 +-- .../sqlite3/oracle/SQLite3CODDTestOracle.java | 69 ++++--------------- 4 files changed, 22 insertions(+), 61 deletions(-) diff --git a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java index 40f8ae20a..246105a5e 100644 --- a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java @@ -313,7 +313,7 @@ public void visit(SQLite3SetClause set) { @Override public void visit(SQLite3Alias alias) { print(alias); - print(alias.getOrigonalExpression()); + print(alias.getOriginalExpression()); print(alias.getAliasExpression()); } diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index 45d9c8db2..a7b392ea7 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -512,7 +512,7 @@ public void visit(SQLite3SetClause set) { @Override public void visit(SQLite3Alias alias) { sb.append("("); - visit(alias.getOrigonalExpression()); + visit(alias.getOriginalExpression()); sb.append(")"); sb.append(" AS "); visit(alias.getAliasExpression()); diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index a14432eec..6c4b2f9ff 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -1593,11 +1593,11 @@ public SQLite3CollateSequence getExplicitCollateSequence() { public static class SQLite3Alias extends SQLite3Expression { - private SQLite3Expression origonalExpression; + private SQLite3Expression originalExpression; private SQLite3Expression aliasExpression; - public SQLite3Alias(SQLite3Expression origonalExpression, SQLite3Expression aliasExpression) { - this.origonalExpression = origonalExpression; + public SQLite3Alias(SQLite3Expression originalExpression, SQLite3Expression aliasExpression) { + this.originalExpression = originalExpression; this.aliasExpression = aliasExpression; } @@ -1606,8 +1606,8 @@ public SQLite3CollateSequence getExplicitCollateSequence() { return null; } - public SQLite3Expression getOrigonalExpression() { - return origonalExpression; + public SQLite3Expression getOriginalExpression() { + return originalExpression; } public SQLite3Expression getAliasExpression() { diff --git a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java index d20e6e9be..7118f4081 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java @@ -100,7 +100,7 @@ public void check() throws SQLException { SQLite3Select auxiliaryQuery = null; if (useSubqueryAsFoldedExpr) { if (useCorrelatedSubqueryAsFoldedExpr) { - auxiliaryQuery = genSelectWithCorrelatedSubquery(null, null); + auxiliaryQuery = genSelectWithCorrelatedSubquery(); auxiliaryQueryString = SQLite3Visitor.asString(auxiliaryQuery); auxiliaryQueryResult.putAll(selectResult); @@ -110,7 +110,7 @@ public void check() throws SQLException { auxiliaryQueryResult = getQueryResult(auxiliaryQueryString, state); } } else { - auxiliaryQuery = genSimpleSelect(null, null); + auxiliaryQuery = genSimpleSelect(); auxiliaryQueryString = SQLite3Visitor.asString(auxiliaryQuery); auxiliaryQueryResult.putAll(selectResult); @@ -334,40 +334,18 @@ else if (Randomly.getBoolean()) { } // For expression test - private SQLite3Select genSimpleSelect(SQLite3Table tempTable, SQLite3Expression specificCondition) { + private SQLite3Select genSimpleSelect() { SQLite3Tables randomTables = s.getRandomTableNonEmptyTables(); - if (tempTable != null) { - randomTables.addTable(tempTable); - } - if (!useSubqueryAsFoldedExpr) { - for (SQLite3Table t : this.tablesFromOuterContext) { - randomTables.addTable(t); - } - if (this.joinsInExpr != null) { - for (Join j : this.joinsInExpr) { - SQLite3Table t = j.getTable(); - randomTables.removeTable(t); - } - } - } - List columns = randomTables.getColumns(); - if (!useSubqueryAsFoldedExpr && this.joinsInExpr != null) { - for (Join j : this.joinsInExpr) { - SQLite3Table t = j.getTable(); - columns.addAll(t.getColumns()); - } - } + gen = new SQLite3ExpressionGenerator(state).setColumns(columns); List tables = randomTables.getTables(); tablesFromOuterContext = randomTables.getTables(); - if (joinsInExpr == null) { - if (Randomly.getBooleanWithRatherLowProbability()) { - joinsInExpr = genJoinExpression(gen, tables, null, true); - } else { - joinsInExpr = new ArrayList(); - } + if (Randomly.getBooleanWithRatherLowProbability()) { + joinsInExpr = genJoinExpression(gen, tables, null, true); + } else { + joinsInExpr = new ArrayList(); } List tableRefs = SQLite3Common.getTableRefs(tables, s); @@ -378,10 +356,6 @@ private SQLite3Select genSimpleSelect(SQLite3Table tempTable, SQLite3Expression } SQLite3Expression whereCondition = gen.generateExpression(); - if (specificCondition != null) { - BinaryOperator operator = BinaryOperator.getRandomOperator(); - whereCondition = new SQLite3Expression.Sqlite3BinaryOperation(whereCondition, specificCondition, operator); - } this.foldedExpr = whereCondition; List fetchColumns = new ArrayList<>(); @@ -401,11 +375,9 @@ private SQLite3Select genSimpleSelect(SQLite3Table tempTable, SQLite3Expression select.setFetchColumns(fetchColumns); - originalQueryString = SQLite3Visitor.asString(select); - Map> queryRes = null; try { - queryRes = getQueryResult(originalQueryString, state); + queryRes = getQueryResult(SQLite3Visitor.asString(select), state); } catch (SQLException e) { if (errors.errorIsExpected(e.getMessage())) { throw new IgnoreMeException(); @@ -429,7 +401,7 @@ private SQLite3Select genSimpleSelect(SQLite3Table tempTable, SQLite3Expression for (int i = 0; i < fetchColumns.size() - 1; ++i) { // do not put the last fetch column to values SQLite3Alias cAlias = (SQLite3Alias) fetchColumns.get(i); - SQLite3ColumnName cRef = (SQLite3ColumnName) cAlias.getOrigonalExpression(); + SQLite3ColumnName cRef = (SQLite3ColumnName) cAlias.getOriginalExpression(); SQLite3Column column = cRef.getColumn(); String columnName = SQLite3Visitor.asString(cAlias.getAliasExpression()); SQLite3Column newColumn = new SQLite3Column(columnName, column.getType(), false, false, null); @@ -448,17 +420,10 @@ private SQLite3Select genSimpleSelect(SQLite3Table tempTable, SQLite3Expression return select; } - private SQLite3Select genSelectWithCorrelatedSubquery(SQLite3Table selectedTable, SQLite3Expression specificCondition) { - // do not support join now - this.joinsInExpr = new ArrayList(); - + private SQLite3Select genSelectWithCorrelatedSubquery() { SQLite3Tables outerQueryRandomTables = s.getRandomTableNonEmptyTables(); SQLite3Tables innerQueryRandomTables = s.getRandomTableNonEmptyTables(); - if (selectedTable != null) { - innerQueryRandomTables.addTable(selectedTable); - } - List innerQueryFromTables = new ArrayList<>(); for (SQLite3Table t : innerQueryRandomTables.getTables()) { if (!outerQueryRandomTables.isContained(t)) { @@ -494,10 +459,6 @@ private SQLite3Select genSelectWithCorrelatedSubquery(SQLite3Table selectedTable innerQuery.setFromList(innerQueryFromTables); SQLite3Expression innerQueryWhereCondition = gen.generateExpression(); - if (specificCondition != null) { - BinaryOperator operator = BinaryOperator.getRandomOperator(); - innerQueryWhereCondition = new SQLite3Expression.Sqlite3BinaryOperation(innerQueryWhereCondition, specificCondition, operator); - } innerQuery.setWhereClause(innerQueryWhereCondition); // use aggregate function in fetch column @@ -506,10 +467,10 @@ private SQLite3Select genSelectWithCorrelatedSubquery(SQLite3Table selectedTable SQLite3Expression innerQueryAggrName = new SQLite3Aggregate(Arrays.asList(innerQueryAggr), SQLite3Aggregate.SQLite3AggregateFunction.getRandom()); innerQuery.setFetchColumns(Arrays.asList(innerQueryAggrName)); if (Randomly.getBooleanWithRatherLowProbability()) { - List groupByClause = genGroupByClause(innerQueryColumns, specificCondition); + List groupByClause = genGroupByClause(innerQueryColumns, null); innerQuery.setGroupByClause(groupByClause); if (groupByClause.size() > 0 && Randomly.getBooleanWithRatherLowProbability()) { - innerQuery.setHavingClause(genHavingClause(innerQueryColumns, specificCondition)); + innerQuery.setHavingClause(genHavingClause(innerQueryColumns, null)); } } @@ -566,7 +527,7 @@ private SQLite3Select genSelectWithCorrelatedSubquery(SQLite3Table selectedTable for (int i = 0; i < outerQueryFetchColumns.size() - 1; ++i) { // do not put the last fetch column to values SQLite3Alias cAlias = (SQLite3Alias) outerQueryFetchColumns.get(i); - SQLite3ColumnName cRef = (SQLite3ColumnName) cAlias.getOrigonalExpression(); + SQLite3ColumnName cRef = (SQLite3ColumnName) cAlias.getOriginalExpression(); SQLite3Column column = cRef.getColumn(); String columnName = SQLite3Visitor.asString(cAlias.getAliasExpression()); SQLite3Column newColumn = new SQLite3Column(columnName, column.getType(), false, false, null); @@ -930,7 +891,7 @@ private Map getColumnTypeFromSelect(SQLite3Select sele for(SQLite3Expression column : fetchColumns) { newFetchColumns.add(column); SQLite3Alias columnAlias = (SQLite3Alias) column; - SQLite3Expression typeofColumn = new SQLite3Typeof(columnAlias.getOrigonalExpression()); + SQLite3Expression typeofColumn = new SQLite3Typeof(columnAlias.getOriginalExpression()); newFetchColumns.add(typeofColumn); } SQLite3Select newSelect = new SQLite3Select(select); From 7a754ba6c9faeadcb2306cdbda7c7a82b018e4ed Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Sun, 5 Jan 2025 21:11:49 +0800 Subject: [PATCH 03/16] add CODDTest for DuckDB --- src/sqlancer/common/ast/SelectBase.java | 31 + .../common/ast/newast/NewExistsNode.java | 19 + .../common/ast/newast/NewToStringVisitor.java | 41 + .../common/ast/newast/NewValuesNode.java | 15 + .../common/ast/newast/NewWithNode.java | 21 + .../gen/UntypedExpressionGenerator.java | 6 +- src/sqlancer/duckdb/DuckDBErrors.java | 19 + src/sqlancer/duckdb/DuckDBOptions.java | 3 + src/sqlancer/duckdb/DuckDBOracleFactory.java | 8 + src/sqlancer/duckdb/DuckDBSchema.java | 13 + .../duckdb/DuckDBToStringVisitor.java | 80 ++ src/sqlancer/duckdb/ast/DuckDBConstant.java | 11 +- .../duckdb/ast/DuckDBConstantWithType.java | 31 + .../duckdb/ast/DuckDBExistsOperator.java | 9 + .../duckdb/ast/DuckDBExpressionBag.java | 17 + src/sqlancer/duckdb/ast/DuckDBResultMap.java | 25 + src/sqlancer/duckdb/ast/DuckDBSelect.java | 14 + src/sqlancer/duckdb/ast/DuckDBTypeCast.java | 19 + src/sqlancer/duckdb/ast/DuckDBTypeofNode.java | 14 + src/sqlancer/duckdb/ast/DuckDBValues.java | 11 + src/sqlancer/duckdb/ast/DuckDBWithClause.java | 11 + .../duckdb/gen/DuckDBExpressionGenerator.java | 9 +- .../duckdb/gen/DuckDBIndexGenerator.java | 2 + .../gen/DuckDBRandomQuerySynthesizer.java | 5 +- .../duckdb/test/DuckDBCODDTestOracle.java | 918 ++++++++++++++++++ 25 files changed, 1344 insertions(+), 8 deletions(-) create mode 100644 src/sqlancer/common/ast/newast/NewExistsNode.java create mode 100644 src/sqlancer/common/ast/newast/NewValuesNode.java create mode 100644 src/sqlancer/common/ast/newast/NewWithNode.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBConstantWithType.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBExistsOperator.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBExpressionBag.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBResultMap.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBTypeCast.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBTypeofNode.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBValues.java create mode 100644 src/sqlancer/duckdb/ast/DuckDBWithClause.java create mode 100644 src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java diff --git a/src/sqlancer/common/ast/SelectBase.java b/src/sqlancer/common/ast/SelectBase.java index e79a1a87d..853d24e05 100644 --- a/src/sqlancer/common/ast/SelectBase.java +++ b/src/sqlancer/common/ast/SelectBase.java @@ -1,5 +1,6 @@ package sqlancer.common.ast; +import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -14,6 +15,21 @@ public class SelectBase { T havingClause; T limitClause; T offsetClause; + T withClause; + + public SelectBase(){} + public SelectBase(SelectBase other) { + fetchColumns = new ArrayList<>(other.fetchColumns); + groupByExpressions = new ArrayList<>(other.groupByExpressions); + orderByExpressions = new ArrayList<>(other.orderByExpressions); + joinList = new ArrayList<>(other.joinList); + fromList = new ArrayList<>(other.fromList); + whereClause = other.whereClause; + havingClause = other.havingClause; + limitClause = other.limitClause; + offsetClause = other.offsetClause; + withClause = other.withClause; + } public void setFetchColumns(List fetchColumns) { if (fetchColumns == null || fetchColumns.isEmpty()) { @@ -40,6 +56,13 @@ public void setFromTables(List tables) { setFromList(tables); } + public void addToFromList(T fromNode) { + if (fromNode == null) { + throw new IllegalArgumentException(); + } + this.fromList.add(fromNode); + } + public List getFromList() { if (fromList == null) { throw new IllegalStateException(); @@ -126,4 +149,12 @@ public List getGroupByClause() { public void setGroupByClause(List groupByExpressions) { setGroupByExpressions(groupByExpressions); } + + public void setWithClause(T withClause) { + this.withClause = withClause; + } + + public T getWithClause() { + return this.withClause; + } } diff --git a/src/sqlancer/common/ast/newast/NewExistsNode.java b/src/sqlancer/common/ast/newast/NewExistsNode.java new file mode 100644 index 000000000..51a3935c5 --- /dev/null +++ b/src/sqlancer/common/ast/newast/NewExistsNode.java @@ -0,0 +1,19 @@ +package sqlancer.common.ast.newast; + +public class NewExistsNode { + private final T expr; + private final Boolean isNot; + + public NewExistsNode(T expr, Boolean isNot) { + this.expr = expr; + this.isNot = isNot; + } + + public T getExpr() { + return expr; + } + + public Boolean getIsNot() { + return isNot; + } +} \ No newline at end of file diff --git a/src/sqlancer/common/ast/newast/NewToStringVisitor.java b/src/sqlancer/common/ast/newast/NewToStringVisitor.java index 82b6bace2..a6621e0a5 100644 --- a/src/sqlancer/common/ast/newast/NewToStringVisitor.java +++ b/src/sqlancer/common/ast/newast/NewToStringVisitor.java @@ -35,12 +35,21 @@ public void visit(E expr) { visit((NewPostfixTextNode) expr); } else if (expr instanceof NewTernaryNode) { visit((NewTernaryNode) expr); + } else if (expr instanceof NewExistsNode) { + visit((NewExistsNode) expr); + } else if (expr instanceof NewValuesNode) { + visit((NewValuesNode) expr); + } else if (expr instanceof NewWithNode) { + visit((NewWithNode) expr); } else { visitSpecific(expr); } } public void visit(List expressions) { + if (expressions.size() > 0 && expressions.get(0) instanceof NewValuesNode) { + sb.append("VALUES "); + } for (int i = 0; i < expressions.size(); i++) { if (i != 0) { sb.append(", "); @@ -59,7 +68,9 @@ public void visit(TableReferenceNode tableRef) { } public void visit(NewAliasNode alias) { + sb.append("("); visit(alias.getExpr()); + sb.append(")"); sb.append(" AS "); sb.append(alias.getAlias()); } @@ -161,6 +172,36 @@ public void visit(NewTernaryNode ternaryNode) { sb.append(")"); } + + public void visit(NewExistsNode existExpr) { + if (existExpr.getIsNot()) { + sb.append(" NOT"); + } + sb.append(" EXISTS("); + visit(existExpr.getExpr()); + sb.append(")"); + } + + public void visit(NewValuesNode valuesExpr) { + sb.append("("); + List values = valuesExpr.getValues(); + for (int i = 0; i < values.size(); ++i) { + if (i != 0) { + sb.append(", "); + } + visit(values.get(i)); + } + sb.append(")"); + } + + public void visit(NewWithNode withExpr) { + sb.append("WITH "); + visit(withExpr.getLeftExpr()); + sb.append(" AS("); + visit(withExpr.getRightExpr()); + sb.append(") "); + } + public String get() { return sb.toString(); } diff --git a/src/sqlancer/common/ast/newast/NewValuesNode.java b/src/sqlancer/common/ast/newast/NewValuesNode.java new file mode 100644 index 000000000..f55c81ff9 --- /dev/null +++ b/src/sqlancer/common/ast/newast/NewValuesNode.java @@ -0,0 +1,15 @@ +package sqlancer.common.ast.newast; + +import java.util.List; + +public class NewValuesNode { + private final List valuesList; + + public NewValuesNode(List valuesList) { + this.valuesList = valuesList; + } + + public List getValues() { + return this.valuesList; + } +} diff --git a/src/sqlancer/common/ast/newast/NewWithNode.java b/src/sqlancer/common/ast/newast/NewWithNode.java new file mode 100644 index 000000000..0f7e7bceb --- /dev/null +++ b/src/sqlancer/common/ast/newast/NewWithNode.java @@ -0,0 +1,21 @@ +package sqlancer.common.ast.newast; + +import java.util.List; + +public class NewWithNode { + private final T leftExpr; + private final List rightExpr; + + public NewWithNode(T leftExpr, List rightExpr) { + this.leftExpr = leftExpr; + this.rightExpr = rightExpr; + } + + public T getLeftExpr() { + return this.leftExpr; + } + + public List getRightExpr() { + return this.rightExpr; + } +} diff --git a/src/sqlancer/common/gen/UntypedExpressionGenerator.java b/src/sqlancer/common/gen/UntypedExpressionGenerator.java index 5a244b53b..51121e58e 100644 --- a/src/sqlancer/common/gen/UntypedExpressionGenerator.java +++ b/src/sqlancer/common/gen/UntypedExpressionGenerator.java @@ -11,7 +11,7 @@ public abstract class UntypedExpressionGenerator implements ExpressionGene protected boolean allowAggregates; public E generateExpression() { - return generateExpression(0); + return generateExpression(Randomly.smallNumber() + 1); } public abstract E generateConstant(); @@ -55,6 +55,10 @@ public List generateOrderBys() { return generateExpressions(Randomly.smallNumber() + 1); } + public List generateGroupBys() { + return generateExpressions(Randomly.smallNumber() + 1); + } + // override this class to generate aggregate functions public E generateHavingClause() { allowAggregates = true; diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index bea136670..ec3f7a337 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -63,6 +63,14 @@ public static List getExpressionErrors() { "FATAL Error: Failed: database has been invalidated because of a previous fatal error. The database must be restarted prior to being used again"); } + // added by CODDTest + errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); + errors.add("must appear in the GROUP BY clause or must be part of an aggregate function"); + errors.add("GROUP BY term out of range - should be between"); + errors.add("INTERNAL Error: Failed to bind column reference"); + errors.add("Binder Error: Aggregate with only constant parameters has to be bound in the root subquery"); + + return errors; } @@ -74,6 +82,9 @@ public static List getExpressionErrorsRegex() { errors.add(Pattern.compile("Cannot mix values of type .* and .* in COALESCE operator")); errors.add(Pattern.compile("Cannot compare values of type .* and type .*")); + // added by CODDTest + // https://github.com/duckdb/duckdb/issues/15554 + errors.add(Pattern.compile("Binder Error: Referenced table \"(.+?)\" not found!")); return errors; } @@ -95,7 +106,9 @@ private static List getFunctionErrors() { errors.add("SUBSTRING cannot handle negative lengths"); errors.add("is undefined outside [-1,1]"); // ACOS etc errors.add("invalid type specifier"); // PRINTF + errors.add("Invalid type specifier"); // PRINTF errors.add("argument index out of range"); // PRINTF + errors.add("Argument index \"0\" out of range"); // PRINTF errors.add("invalid format string"); // PRINTF errors.add("number is too big"); // PRINTF errors.add("Like pattern must not end with escape character!"); // LIKE @@ -135,6 +148,12 @@ public static List getInsertErrors() { errors.add("create unique index, table contains duplicate data"); errors.add("Failed to cast"); + // added by CODDTest + errors.add("violates primary key constraint"); + errors.add("Referenced update column rowid not found in table"); + errors.add("does not have a column with name \"rowid\""); + errors.add("violates unique constraint"); + return errors; } diff --git a/src/sqlancer/duckdb/DuckDBOptions.java b/src/sqlancer/duckdb/DuckDBOptions.java index 00f85eece..811e43c47 100644 --- a/src/sqlancer/duckdb/DuckDBOptions.java +++ b/src/sqlancer/duckdb/DuckDBOptions.java @@ -77,6 +77,9 @@ public class DuckDBOptions implements DBMSSpecificOptions { @Parameter(names = "--max-num-updates", description = "The maximum number of UPDATE statements that are issued for a database", arity = 1) public int maxNumUpdates = 5; + @Parameter(names = { "--coddtest-model" }, description = "Apply CODDTest on expression, subquery, or random") + public String coddTestModel = "random"; + @Parameter(names = "--oracle") public List oracles = Arrays.asList(DuckDBOracleFactory.QUERY_PARTITIONING); diff --git a/src/sqlancer/duckdb/DuckDBOracleFactory.java b/src/sqlancer/duckdb/DuckDBOracleFactory.java index 8fd0f96af..a4c2fd82c 100644 --- a/src/sqlancer/duckdb/DuckDBOracleFactory.java +++ b/src/sqlancer/duckdb/DuckDBOracleFactory.java @@ -10,7 +10,9 @@ import sqlancer.common.oracle.TLPWhereOracle; import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; +import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; import sqlancer.duckdb.gen.DuckDBExpressionGenerator; +import sqlancer.duckdb.test.DuckDBCODDTestOracle; import sqlancer.duckdb.test.DuckDBQueryPartitioningAggregateTester; import sqlancer.duckdb.test.DuckDBQueryPartitioningDistinctTester; import sqlancer.duckdb.test.DuckDBQueryPartitioningGroupByTester; @@ -81,6 +83,12 @@ public TestOracle create(DuckDBProvider.DuckDB oracles.add(GROUP_BY.create(globalState)); return new CompositeTestOracle(oracles, globalState); } + }, + CODDTest { + @Override + public TestOracle create(DuckDBGlobalState globalState) throws SQLException { + return new DuckDBCODDTestOracle(globalState); + } }; } diff --git a/src/sqlancer/duckdb/DuckDBSchema.java b/src/sqlancer/duckdb/DuckDBSchema.java index 379576fd8..2bff60398 100644 --- a/src/sqlancer/duckdb/DuckDBSchema.java +++ b/src/sqlancer/duckdb/DuckDBSchema.java @@ -41,9 +41,19 @@ public static class DuckDBCompositeDataType { private final int size; + // This is used to handle the type that out of the scope of our code + private final String typeName; + public DuckDBCompositeDataType(DuckDBDataType dataType, int size) { this.dataType = dataType; this.size = size; + this.typeName = ""; + } + + public DuckDBCompositeDataType(String typeName) { + this.dataType = null; + this.size = 0; + this.typeName = typeName; } public DuckDBDataType getPrimitiveDataType() { @@ -82,6 +92,9 @@ public static DuckDBCompositeDataType getRandomWithoutNull() { @Override public String toString() { + if (getPrimitiveDataType() == null) { + return this.typeName; + } switch (getPrimitiveDataType()) { case INT: switch (size) { diff --git a/src/sqlancer/duckdb/DuckDBToStringVisitor.java b/src/sqlancer/duckdb/DuckDBToStringVisitor.java index b1a8eb703..e43cd0d3b 100644 --- a/src/sqlancer/duckdb/DuckDBToStringVisitor.java +++ b/src/sqlancer/duckdb/DuckDBToStringVisitor.java @@ -1,11 +1,21 @@ package sqlancer.duckdb; +import java.util.LinkedHashMap; +import java.util.List; + import sqlancer.common.ast.newast.NewToStringVisitor; import sqlancer.common.ast.newast.TableReferenceNode; +import sqlancer.duckdb.ast.DuckDBColumnReference; import sqlancer.duckdb.ast.DuckDBConstant; +import sqlancer.duckdb.ast.DuckDBConstantWithType; import sqlancer.duckdb.ast.DuckDBExpression; +import sqlancer.duckdb.ast.DuckDBExpressionBag; import sqlancer.duckdb.ast.DuckDBJoin; +import sqlancer.duckdb.ast.DuckDBResultMap; import sqlancer.duckdb.ast.DuckDBSelect; +import sqlancer.duckdb.ast.DuckDBTypeCast; +import sqlancer.duckdb.ast.DuckDBTypeofNode; +import sqlancer.duckdb.ast.DuckDBConstant.DuckDBNullConstant; public class DuckDBToStringVisitor extends NewToStringVisitor { @@ -17,6 +27,16 @@ public void visitSpecific(DuckDBExpression expr) { visit((DuckDBSelect) expr); } else if (expr instanceof DuckDBJoin) { visit((DuckDBJoin) expr); + } else if (expr instanceof DuckDBConstantWithType) { + visit((DuckDBConstantWithType) expr); + } else if (expr instanceof DuckDBResultMap) { + visit((DuckDBResultMap) expr); + } else if (expr instanceof DuckDBTypeCast) { + visit((DuckDBTypeCast) expr); + } else if (expr instanceof DuckDBTypeofNode) { + visit((DuckDBTypeofNode) expr); + } else if (expr instanceof DuckDBExpressionBag) { + visit((DuckDBExpressionBag) expr); } else { throw new AssertionError(expr.getClass()); } @@ -39,10 +59,15 @@ private void visit(DuckDBJoin join) { } private void visit(DuckDBConstant constant) { + sb.append("("); sb.append(constant.toString()); + sb.append(")"); } private void visit(DuckDBSelect select) { + if (select.getWithClause() != null) { + visit(select.getWithClause()); + } sb.append("SELECT "); if (select.isDistinct()) { sb.append("DISTINCT "); @@ -82,10 +107,65 @@ private void visit(DuckDBSelect select) { } } + public void visit(DuckDBConstantWithType expr) { + sb.append(expr.toString()); + } + + public void visit(DuckDBResultMap expr) { + // use CASE WHEN to express the constant result of a expression + LinkedHashMap> dbstate = expr.getDbStates(); + List result = expr.getResult(); + int size = dbstate.values().iterator().next().size(); + if (size == 0) { + sb.append(" NULL "); + return; + } + sb.append(" CASE "); + for (int i = 0; i < size; i++) { + sb.append("WHEN "); + for (DuckDBColumnReference columnRef : dbstate.keySet()) { + visit(columnRef); + if (dbstate.get(columnRef).get(i) instanceof DuckDBNullConstant) { + sb.append(" IS NULL"); + } else { + sb.append(" = "); + visit(dbstate.get(columnRef).get(i)); + } + sb.append(" AND "); + } + for (int k = 0; k < 4; k++) { + sb.deleteCharAt(sb.length() - 1); + } + sb.append("THEN "); + visit(result.get(i)); + sb.append(" "); + } + sb.append("END "); + } + + public void visit(DuckDBTypeCast expr) { + sb.append("("); + visit(expr.getExpression()); + sb.append(")::"); + sb.append(expr.getType().toString()); + sb.append(" "); + } + + public void visit(DuckDBTypeofNode expr) { + sb.append("typeof("); + visit(expr.getExpr()); + sb.append(")"); + } + + public void visit(DuckDBExpressionBag expr) { + visit(expr.getInnerExpr()); + } + public static String asString(DuckDBExpression expr) { DuckDBToStringVisitor visitor = new DuckDBToStringVisitor(); visitor.visit(expr); return visitor.get(); } + } diff --git a/src/sqlancer/duckdb/ast/DuckDBConstant.java b/src/sqlancer/duckdb/ast/DuckDBConstant.java index f4af6918a..b19d808d2 100644 --- a/src/sqlancer/duckdb/ast/DuckDBConstant.java +++ b/src/sqlancer/duckdb/ast/DuckDBConstant.java @@ -1,5 +1,6 @@ package sqlancer.duckdb.ast; +import java.math.BigInteger; import java.sql.Timestamp; import java.text.SimpleDateFormat; @@ -19,9 +20,9 @@ public String toString() { public static class DuckDBIntConstant extends DuckDBConstant { - private final long value; + private final BigInteger value; - public DuckDBIntConstant(long value) { + public DuckDBIntConstant(BigInteger value) { this.value = value; } @@ -30,7 +31,7 @@ public String toString() { return String.valueOf(value); } - public long getValue() { + public BigInteger getValue() { return value; } @@ -54,6 +55,8 @@ public String toString() { return "'+Inf'"; } else if (value == Double.NEGATIVE_INFINITY) { return "'-Inf'"; + } else if (Double.isNaN(value)) { + return "'NaN'"; } return String.valueOf(value); } @@ -167,7 +170,7 @@ public static DuckDBExpression createFloatConstant(double val) { return new DuckDBDoubleConstant(val); } - public static DuckDBExpression createIntConstant(long val) { + public static DuckDBExpression createIntConstant(BigInteger val) { return new DuckDBIntConstant(val); } diff --git a/src/sqlancer/duckdb/ast/DuckDBConstantWithType.java b/src/sqlancer/duckdb/ast/DuckDBConstantWithType.java new file mode 100644 index 000000000..f6c43cc3f --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBConstantWithType.java @@ -0,0 +1,31 @@ +package sqlancer.duckdb.ast; + +import sqlancer.duckdb.DuckDBSchema.DuckDBCompositeDataType; +import sqlancer.duckdb.ast.DuckDBConstant.DuckDBNullConstant; + +public class DuckDBConstantWithType implements DuckDBExpression { + private DuckDBConstant constant; + private DuckDBCompositeDataType type; + + public DuckDBConstantWithType(DuckDBConstant constant, DuckDBCompositeDataType type) { + this.constant = constant; + this.type = type; + } + + public String toString() { + if (constant instanceof DuckDBNullConstant) { + return constant.toString(); + } + else { + return "(" + constant.toString() + ")::" + type.toString(); + } + } + + public DuckDBConstant getConstant() { + return constant; + } + + public DuckDBCompositeDataType getType() { + return type; + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBExistsOperator.java b/src/sqlancer/duckdb/ast/DuckDBExistsOperator.java new file mode 100644 index 000000000..3144b1c51 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBExistsOperator.java @@ -0,0 +1,9 @@ +package sqlancer.duckdb.ast; + +import sqlancer.common.ast.newast.NewExistsNode; + +public class DuckDBExistsOperator extends NewExistsNode implements DuckDBExpression { + public DuckDBExistsOperator(DuckDBExpression expr, Boolean isNot) { + super(expr, isNot); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBExpressionBag.java b/src/sqlancer/duckdb/ast/DuckDBExpressionBag.java new file mode 100644 index 000000000..29e31cb19 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBExpressionBag.java @@ -0,0 +1,17 @@ +package sqlancer.duckdb.ast; + +public class DuckDBExpressionBag implements DuckDBExpression { + DuckDBExpression expr; + + public DuckDBExpressionBag(DuckDBExpression e) { + this.expr = e; + } + + public void updateInnerExpr(DuckDBExpression expr) { + this.expr = expr; + } + + public DuckDBExpression getInnerExpr() { + return this.expr; + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBResultMap.java b/src/sqlancer/duckdb/ast/DuckDBResultMap.java new file mode 100644 index 000000000..03e9b23a3 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBResultMap.java @@ -0,0 +1,25 @@ +package sqlancer.duckdb.ast; + +import java.util.LinkedHashMap; +import java.util.List; + +public class DuckDBResultMap implements DuckDBExpression { + private final LinkedHashMap> DBStates; + private final List results; + + public DuckDBResultMap(LinkedHashMap> s, List r) { + this.DBStates = s; + this.results = r; + if (s.get(s.keySet().iterator().next()).size() != r.size()) { + throw new AssertionError(); + } + } + + public LinkedHashMap> getDbStates() { + return this.DBStates; + } + + public List getResult() { + return this.results; + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBSelect.java b/src/sqlancer/duckdb/ast/DuckDBSelect.java index e18e57a4d..594987585 100644 --- a/src/sqlancer/duckdb/ast/DuckDBSelect.java +++ b/src/sqlancer/duckdb/ast/DuckDBSelect.java @@ -38,4 +38,18 @@ public List getJoinClauses() { public String asString() { return DuckDBToStringVisitor.asString(this); } + + public DuckDBSelect() {} + public DuckDBSelect(DuckDBSelect oldSelect) { + this.setWithClause(oldSelect.getWithClause()); + this.setFetchColumns(oldSelect.getFetchColumns()); + this.setFromList(oldSelect.getFromList()); + this.setJoinList(oldSelect.getJoinList()); + this.setWhereClause(oldSelect.getWhereClause()); + this.setOrderByClauses(oldSelect.getOrderByClauses()); + this.setGroupByExpressions(oldSelect.getGroupByExpressions()); + this.setHavingClause(oldSelect.getHavingClause()); + this.setLimitClause(oldSelect.getLimitClause()); + this.setOffsetClause(oldSelect.getOffsetClause()); + } } diff --git a/src/sqlancer/duckdb/ast/DuckDBTypeCast.java b/src/sqlancer/duckdb/ast/DuckDBTypeCast.java new file mode 100644 index 000000000..5cdd0e1bd --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBTypeCast.java @@ -0,0 +1,19 @@ +package sqlancer.duckdb.ast; +import sqlancer.duckdb.DuckDBSchema.DuckDBCompositeDataType; + +public class DuckDBTypeCast implements DuckDBExpression { + DuckDBExpression expr; + DuckDBCompositeDataType type; + public DuckDBTypeCast(DuckDBExpression e, DuckDBCompositeDataType t) { + this.expr = e; + this.type = t; + } + + public DuckDBExpression getExpression() { + return this.expr; + } + + public DuckDBCompositeDataType getType() { + return this.type; + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBTypeofNode.java b/src/sqlancer/duckdb/ast/DuckDBTypeofNode.java new file mode 100644 index 000000000..26c7fc1a5 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBTypeofNode.java @@ -0,0 +1,14 @@ +package sqlancer.duckdb.ast; + +public class DuckDBTypeofNode implements DuckDBExpression { + + private final DuckDBExpression expr; + + public DuckDBTypeofNode(DuckDBExpression e) { + this.expr = e; + } + + public DuckDBExpression getExpr() { + return expr; + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBValues.java b/src/sqlancer/duckdb/ast/DuckDBValues.java new file mode 100644 index 000000000..d98e2eeb5 --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBValues.java @@ -0,0 +1,11 @@ +package sqlancer.duckdb.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewValuesNode; + +public class DuckDBValues extends NewValuesNode implements DuckDBExpression { + public DuckDBValues(List exprs) { + super(exprs); + } +} diff --git a/src/sqlancer/duckdb/ast/DuckDBWithClause.java b/src/sqlancer/duckdb/ast/DuckDBWithClause.java new file mode 100644 index 000000000..fb7118f2e --- /dev/null +++ b/src/sqlancer/duckdb/ast/DuckDBWithClause.java @@ -0,0 +1,11 @@ +package sqlancer.duckdb.ast; + +import java.util.List; + +import sqlancer.common.ast.newast.NewWithNode; + +public class DuckDBWithClause extends NewWithNode implements DuckDBExpression { + public DuckDBWithClause(DuckDBExpression left, List right) { + super(left, right); + } +} diff --git a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java index 278685a46..2cd0ce71f 100644 --- a/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBExpressionGenerator.java @@ -1,5 +1,6 @@ package sqlancer.duckdb.gen; +import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -147,7 +148,7 @@ public DuckDBExpression generateConstant() { if (!globalState.getDbmsSpecificOptions().testIntConstants) { throw new IgnoreMeException(); } - return DuckDBConstant.createIntConstant(globalState.getRandomly().getInteger()); + return DuckDBConstant.createIntConstant(BigInteger.valueOf(globalState.getRandomly().getInteger())); case DATE: if (!globalState.getDbmsSpecificOptions().testDateConstants) { throw new IgnoreMeException(); @@ -191,6 +192,12 @@ public List generateOrderBys() { return newExpr; }; + @Override + public List generateGroupBys() { + List expr = super.generateGroupBys(); + return expr; + }; + public static class DuckDBCastOperation extends NewUnaryPostfixOperatorNode implements DuckDBExpression { diff --git a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java index 80bbfd61c..126e368b6 100644 --- a/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java +++ b/src/sqlancer/duckdb/gen/DuckDBIndexGenerator.java @@ -44,6 +44,8 @@ public static SQLQueryAdapter getQuery(DuckDBGlobalState globalState) { if (globalState.getDbmsSpecificOptions().testRowid) { errors.add("Cannot create an index on the rowid!"); } + + errors.add("Data contains duplicates on indexed column(s)"); // added by CODDTest return new SQLQueryAdapter(sb.toString(), errors, true); } diff --git a/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java b/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java index d88d4f0b8..117242281 100644 --- a/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java +++ b/src/sqlancer/duckdb/gen/DuckDBRandomQuerySynthesizer.java @@ -1,5 +1,6 @@ package sqlancer.duckdb.gen; +import java.math.BigInteger; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -54,11 +55,11 @@ public static DuckDBSelect generateSelect(DuckDBGlobalState globalState, int nrC } if (Randomly.getBoolean()) { - select.setLimitClause(DuckDBConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + select.setLimitClause(DuckDBConstant.createIntConstant(BigInteger.valueOf(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)))); } if (Randomly.getBoolean()) { select.setOffsetClause( - DuckDBConstant.createIntConstant(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE))); + DuckDBConstant.createIntConstant(BigInteger.valueOf(Randomly.getNotCachedInteger(0, Integer.MAX_VALUE)))); } if (Randomly.getBoolean()) { select.setHavingClause(gen.generateHavingClause()); diff --git a/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java b/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java new file mode 100644 index 000000000..a102a9ee1 --- /dev/null +++ b/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java @@ -0,0 +1,918 @@ +package sqlancer.duckdb.test; + +import java.math.BigInteger; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Main; +import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.common.ast.BinaryOperatorNode.Operator; +import sqlancer.common.ast.newast.NewOrderingTerm.Ordering; +import sqlancer.common.oracle.CODDTestBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.duckdb.DuckDBErrors; +import sqlancer.duckdb.DuckDBSchema; +import sqlancer.duckdb.DuckDBToStringVisitor; +import sqlancer.duckdb.DuckDBProvider.DuckDBGlobalState; +import sqlancer.duckdb.DuckDBSchema.DuckDBColumn; +import sqlancer.duckdb.DuckDBSchema.DuckDBCompositeDataType; +import sqlancer.duckdb.DuckDBSchema.DuckDBTable; +import sqlancer.duckdb.DuckDBSchema.DuckDBTables; +import sqlancer.duckdb.ast.DuckDBAlias; +import sqlancer.duckdb.ast.DuckDBBinaryOperator; +import sqlancer.duckdb.ast.DuckDBColumnReference; +import sqlancer.duckdb.ast.DuckDBConstant; +import sqlancer.duckdb.ast.DuckDBConstantWithType; +import sqlancer.duckdb.ast.DuckDBExistsOperator; +import sqlancer.duckdb.ast.DuckDBExpression; +import sqlancer.duckdb.ast.DuckDBExpressionBag; +import sqlancer.duckdb.ast.DuckDBFunction; +import sqlancer.duckdb.ast.DuckDBInOperator; +import sqlancer.duckdb.ast.DuckDBJoin; +import sqlancer.duckdb.ast.DuckDBSelect; +import sqlancer.duckdb.ast.DuckDBTableReference; +import sqlancer.duckdb.ast.DuckDBTypeCast; +import sqlancer.duckdb.ast.DuckDBTypeofNode; +import sqlancer.duckdb.ast.DuckDBValues; +import sqlancer.duckdb.ast.DuckDBWithClause; +import sqlancer.duckdb.ast.DuckDBConstant.DuckDBTextConstant; +import sqlancer.duckdb.ast.DuckDBJoin.OuterType; +import sqlancer.duckdb.ast.DuckDBOrderingTerm; +import sqlancer.duckdb.ast.DuckDBResultMap; +import sqlancer.duckdb.gen.DuckDBExpressionGenerator; +import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBAggregateFunction; +import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBBinaryArithmeticOperator; +import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBBinaryComparisonOperator; +import sqlancer.duckdb.gen.DuckDBExpressionGenerator.DuckDBBinaryLogicalOperator; + + +public class DuckDBCODDTestOracle extends CODDTestBase implements TestOracle { + + private final DuckDBSchema s; + private DuckDBExpressionGenerator gen; + private Reproducer reproducer; + + private String tempTableName = "temp_table"; + + private DuckDBExpression foldedExpr; + private DuckDBExpression constantResOfFoldedExpr; + + private List tablesFromOuterContext = new ArrayList<>(); + private List joinsInExpr = null; + + Map> auxiliaryQueryResult = new HashMap<>(); + Map> selectResult = new HashMap<>(); + + Boolean useSubqueryAsFoldedExpr; + Boolean useCorrelatedSubqueryAsFoldedExpr; + + public DuckDBCODDTestOracle(DuckDBGlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + DuckDBErrors.addExpressionErrors(errors); + DuckDBErrors.addInsertErrors(errors); + DuckDBErrors.addGroupByErrors(errors); + } + + @Override + public void check() throws Exception { + reproducer = null; + + useSubqueryAsFoldedExpr = useSubquery(); + useCorrelatedSubqueryAsFoldedExpr = useCorrelatedSubquery(); + + DuckDBSelect auxiliaryQuery = null; + + if (useSubqueryAsFoldedExpr) { + if (useCorrelatedSubqueryAsFoldedExpr) { + auxiliaryQuery = genSelectWithCorrelatedSubquery(); + auxiliaryQueryString = DuckDBToStringVisitor.asString(auxiliaryQuery); + auxiliaryQueryResult.putAll(selectResult); + } else { + auxiliaryQuery = genSelectExpression(null, null); + auxiliaryQueryString = DuckDBToStringVisitor.asString(auxiliaryQuery); + auxiliaryQueryResult = getQueryResult(auxiliaryQueryString, state); + } + } else { + auxiliaryQuery = genSimpleSelect(); + auxiliaryQueryString = DuckDBToStringVisitor.asString(auxiliaryQuery); + auxiliaryQueryResult.putAll(selectResult); + } + + DuckDBSelect originalQuery = null; + + Map> foldedResult = null; + Map> originalResult = null; + + // dependent expression + if (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) { + DuckDBExpressionBag specificCondition = new DuckDBExpressionBag(this.foldedExpr); + originalQuery = this.genSelectExpression(null, specificCondition); + originalQueryString = DuckDBToStringVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + specificCondition.updateInnerExpr(this.constantResOfFoldedExpr); + foldedQueryString = DuckDBToStringVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + // independent expression + // empty result, put the inner query in (NOT) EXIST + else if (auxiliaryQueryResult.size() == 0 || auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().iterator().next()).size() == 0) { + boolean isNegated = Randomly.getBoolean() ? false : true; + + // original query + DuckDBExistsOperator existExpr = new DuckDBExistsOperator(auxiliaryQuery, isNegated); + DuckDBExpressionBag specificCondition = new DuckDBExpressionBag(existExpr); + originalQuery = this.genSelectExpression(null, specificCondition); + originalQueryString = DuckDBToStringVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + DuckDBExpression equivalentExpr = isNegated ? DuckDBConstant.createBooleanConstant(true) : DuckDBConstant.createBooleanConstant(false); + specificCondition.updateInnerExpr(equivalentExpr); + foldedQueryString = DuckDBToStringVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + // Scalar Subquery: 1 column and 1 row, consider the inner query as a constant + else if (auxiliaryQueryResult.size() == 1 && auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]).size() == 1 && Randomly.getBoolean()) { + // float value is inexact, as https://duckdb.org/docs/sql/data_types/numeric#floating-point-types + String typeName = getColumnTypeFromSelect(auxiliaryQuery).get(0).toString(); + if (typeName.startsWith("FLOAT") || typeName.startsWith("DOUBLE") || typeName.startsWith("REAL")) { + throw new IgnoreMeException(); + } + // original query + DuckDBExpressionBag specificCondition = new DuckDBExpressionBag(auxiliaryQuery); + originalQuery = this.genSelectExpression(null, specificCondition); + originalQueryString = DuckDBToStringVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + DuckDBCompositeDataType constantType = this.getColumnTypeFromSelect(auxiliaryQuery).get(0); + DuckDBConstant constant = (DuckDBConstant) auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]).get(0); + DuckDBConstantWithType equivalentExpr = new DuckDBConstantWithType(constant, constantType); + specificCondition.updateInnerExpr(equivalentExpr); + foldedQueryString = DuckDBToStringVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + // one column + else if (auxiliaryQueryResult.size() == 1 && Randomly.getBoolean()) { + // float value is inexact, as https://duckdb.org/docs/sql/data_types/numeric#floating-point-types + DuckDBCompositeDataType valuesType = getColumnTypeFromSelect(auxiliaryQuery).get(0); + String typeName = valuesType.toString(); + if (typeName.startsWith("FLOAT") || typeName.startsWith("DOUBLE") || typeName.startsWith("REAL")) { + throw new IgnoreMeException(); + } + // original query + List columns = s.getRandomTableNonEmptyTables().getColumns(); + DuckDBColumnReference selectedColumn = new DuckDBColumnReference(Randomly.fromList(columns)); + DuckDBTable selectedTable = selectedColumn.getColumn().getTable(); + DuckDBTableReference selectedTableRef = new DuckDBTableReference(selectedTable); + DuckDBExpressionBag tableBag = new DuckDBExpressionBag(selectedTableRef); + + Boolean isNegatedIn = Randomly.getBoolean(); + DuckDBTypeCast columnWithType = new DuckDBTypeCast(selectedColumn, valuesType); + DuckDBInOperator optInOperation = new DuckDBInOperator(columnWithType, Arrays.asList(auxiliaryQuery), isNegatedIn); + DuckDBExpressionBag specificCondition = new DuckDBExpressionBag(optInOperation); + originalQuery = this.genSelectExpression(tableBag, specificCondition); + originalQueryString = DuckDBToStringVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + List values = new ArrayList<>(); + Map typeList = this.getColumnTypeFromSelect(auxiliaryQuery); + for (int i = 0; i < auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]).size(); ++i) { + List rowRs = new ArrayList<>(); + for (int j = 0; j < typeList.size(); ++j) { + DuckDBConstant c = (DuckDBConstant) auxiliaryQueryResult.get("c" + String.valueOf(j)).get(i); + rowRs.add(new DuckDBConstantWithType(c, typeList.get(j))); + } + DuckDBValues valueRow = new DuckDBValues(rowRs); + values.add(valueRow); + } + + DuckDBInOperator refInOperation = new DuckDBInOperator(columnWithType, values, isNegatedIn); + specificCondition.updateInnerExpr(refInOperation); + foldedQueryString = DuckDBToStringVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + // Row Subquery + else { + // original query + DuckDBTable temporaryTable = this.genTemporaryTable(auxiliaryQuery, this.tempTableName); + DuckDBTableReference tableRef = new DuckDBTableReference(temporaryTable); + DuckDBExpressionBag tableBag = new DuckDBExpressionBag(tableRef); + originalQuery = this.genSelectExpression(tableBag, null); + + DuckDBWithClause optWithClasure = new DuckDBWithClause(tableRef, Arrays.asList(auxiliaryQuery)); + originalQuery.setWithClause(optWithClasure); + originalQueryString = DuckDBToStringVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + if (Randomly.getBoolean()) { + // folded query: WITH table AS VALUES () + List values = new ArrayList<>(); + Map typeList = this.getColumnTypeFromSelect(auxiliaryQuery); + for (int i = 0; i < auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]).size(); ++i){ + List rowRs = new ArrayList<>(); + for (int j = 0; j < typeList.size(); ++j) { + DuckDBConstant c = (DuckDBConstant) auxiliaryQueryResult.get("c" + String.valueOf(j)).get(i); + rowRs.add(new DuckDBConstantWithType(c, typeList.get(j))); + } + DuckDBValues valueRow = new DuckDBValues(rowRs); + values.add(valueRow); + } + DuckDBWithClause refWithClasure = new DuckDBWithClause(tableRef, values); + originalQuery.setWithClause(refWithClasure); + foldedQueryString = DuckDBToStringVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } else if (Randomly.getBoolean()) { + // folded query: SELECT FROM () AS table + originalQuery.setWithClause(null); + DuckDBAlias alias = new DuckDBAlias(auxiliaryQuery, DuckDBToStringVisitor.asString(tableRef)); + tableBag.updateInnerExpr(alias); + foldedQueryString = DuckDBToStringVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } else { + // folded query: CREATE the table + try { + this.createTemporaryTable(auxiliaryQuery, this.tempTableName); + originalQuery.setWithClause(null); + foldedQueryString = DuckDBToStringVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } finally { + dropTemporaryTable(this.tempTableName); + } + } + } + + if (foldedResult == null || originalResult == null) { + throw new IgnoreMeException(); + } + + if (foldedQueryString.equals(originalQueryString)) { + throw new IgnoreMeException(); + } + + if (!this.compareResult(foldedResult, originalResult)) { + reproducer = null; + state.getState().getLocalState().log(auxiliaryQueryString + ";\n" + foldedQueryString + ";\n" + originalQueryString + ";"); + throw new AssertionError(auxiliaryQueryResult.toString() + " " + foldedResult.toString() + " " + originalResult.toString()); + } + + } + + private DuckDBSelect genSelectExpression(DuckDBExpressionBag tableBag, DuckDBExpression specificCondition) { + DuckDBTables randomTables = s.getRandomTableNonEmptyTables(); + if (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) { + for (DuckDBTable t : this.tablesFromOuterContext) { + randomTables.addTable(t); + } + if (this.joinsInExpr != null) { + for (DuckDBJoin j : this.joinsInExpr) { + DuckDBTable t = j.getRightTable().getTable(); + randomTables.removeTable(t); + } + for (DuckDBJoin j : this.joinsInExpr) { + DuckDBTable t = j.getLeftTable().getTable(); + randomTables.removeTable(t); + } + } + } + DuckDBTable tempTable = null; + DuckDBTableReference tableRef = null; + if (tableBag != null) { + tableRef = (DuckDBTableReference) tableBag.getInnerExpr(); + tempTable = tableRef.getTable(); + // randomTables.addTable(tempTable); + } + List columns = randomTables.getColumns(); + if (tempTable != null) { + columns.addAll(tempTable.getColumns()); + } + if ((!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) && this.joinsInExpr != null) { + for (DuckDBJoin j : this.joinsInExpr) { + DuckDBTable t = j.getRightTable().getTable(); + columns.addAll(t.getColumns()); + t = j.getLeftTable().getTable(); + columns.addAll(t.getColumns()); + } + } + gen = new DuckDBExpressionGenerator(state).setColumns(columns); + List tables = randomTables.getTables(); + List tableRefList = tables.stream() + .map(t -> new DuckDBTableReference(t)).collect(Collectors.toList()); + + DuckDBSelect select = new DuckDBSelect(); + + List joins = new ArrayList<>(); + if ((!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) && this.joinsInExpr != null) { + joins.addAll(this.joinsInExpr); + this.joinsInExpr = null; + } + else if (Randomly.getBoolean()) { + joins = genJoinExpression(tableRefList, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null); + } + if (joins.size() > 0) { + select.setJoinList(joins); + } + + select.setFromList(tableRefList.stream().collect(Collectors.toList())); + if (tableBag != null && !tables.contains(tempTable)) { + select.addToFromList(tableBag); + } + + DuckDBExpression randomWhereCondition = gen.generateExpression(); + if (specificCondition != null) { + Operator operator = Randomly.fromList(Arrays.asList(DuckDBBinaryComparisonOperator.getRandom(), DuckDBBinaryArithmeticOperator.getRandom(), DuckDBBinaryLogicalOperator.getRandom())); + randomWhereCondition = new DuckDBBinaryOperator(randomWhereCondition, specificCondition, operator); + } + select.setWhereClause(randomWhereCondition); + + if (Randomly.getBoolean()) { + select.setOrderByClauses(genOrderBys(specificCondition)); + } + + if (Randomly.getBoolean()) { + List selectedColumns = Randomly.nonEmptySubset(columns); + List selectedAlias = new ArrayList<>(); + for (int i = 0; i < selectedColumns.size(); ++i) { + DuckDBColumnReference originalName = new DuckDBColumnReference(selectedColumns.get(i)); + DuckDBAlias columnAlias = new DuckDBAlias(originalName, "c" + String.valueOf(i)); + selectedAlias.add(columnAlias); + } + select.setFetchColumns(selectedAlias); + } else { + DuckDBColumnReference columnName = new DuckDBColumnReference(Randomly.fromList(columns)); + DuckDBAggregateFunction aggregateFunction = Randomly.fromOptions(DuckDBAggregateFunction.MAX, + DuckDBAggregateFunction.MIN, DuckDBAggregateFunction.SUM, DuckDBAggregateFunction.COUNT, + DuckDBAggregateFunction.AVG); + DuckDBFunction aggregate = new DuckDBFunction<>(Arrays.asList(columnName), aggregateFunction); + DuckDBAlias columnAlias = new DuckDBAlias(aggregate, "c0"); + select.setFetchColumns(Arrays.asList(columnAlias)); + select.setGroupByExpressions(genGroupBys(Randomly.nonEmptySubset(columns), specificCondition)); + // gen having + if (Randomly.getBooleanWithRatherLowProbability()) { + DuckDBExpressionGenerator havingGen = new DuckDBExpressionGenerator(state).setColumns(columns); + DuckDBExpression havingExpr = havingGen.generateExpression(); + if (specificCondition != null && Randomly.getBooleanWithRatherLowProbability()) { + Operator operator = Randomly.fromList(Arrays.asList(DuckDBBinaryComparisonOperator.getRandom(), DuckDBBinaryArithmeticOperator.getRandom(), DuckDBBinaryLogicalOperator.getRandom())); + havingExpr = new DuckDBBinaryOperator(havingExpr, specificCondition, operator); + } + select.setHavingClause(havingExpr); + } + } + return select; + } + + private DuckDBSelect genSelectWithCorrelatedSubquery() { + DuckDBTables outerQueryRandomTables = s.getRandomTableNonEmptyTables(); + DuckDBTables innerQueryRandomTables = s.getRandomTableNonEmptyTables(); + + List innerQueryFromTables = new ArrayList<>(); + for (DuckDBTable t : innerQueryRandomTables.getTables()) { + if (!outerQueryRandomTables.isContained(t)) + innerQueryFromTables.add(new DuckDBTableReference(t)); + } + if (innerQueryFromTables.size() == 0) { + throw new IgnoreMeException(); + } + + for (DuckDBTable t : outerQueryRandomTables.getTables()) { + if (!innerQueryRandomTables.isContained(t)) { + tablesFromOuterContext.add(t); + } + } + + List innerQueryColumns = new ArrayList<>(); + innerQueryColumns.addAll(innerQueryRandomTables.getColumns()); + innerQueryColumns.addAll(outerQueryRandomTables.getColumns()); + gen = new DuckDBExpressionGenerator(state).setColumns(innerQueryColumns); + + DuckDBSelect innerQuery = new DuckDBSelect(); + innerQuery.setFromList(innerQueryFromTables); + + DuckDBExpression innerQueryWhereCondition = gen.generateExpression(); + innerQuery.setWhereClause(innerQueryWhereCondition); + + // use aggregate function in fetch column + DuckDBColumnReference innerQueryAggrColumn = new DuckDBColumnReference(Randomly.fromList(innerQueryRandomTables.getColumns())); + DuckDBAggregateFunction aggregateFunction = Randomly.fromOptions(DuckDBAggregateFunction.MAX, + DuckDBAggregateFunction.MIN, DuckDBAggregateFunction.SUM, DuckDBAggregateFunction.COUNT, + DuckDBAggregateFunction.AVG); + DuckDBFunction aggregate = new DuckDBFunction<>(Arrays.asList(innerQueryAggrColumn), aggregateFunction); + innerQuery.setFetchColumns(Arrays.asList(aggregate)); + if (Randomly.getBooleanWithRatherLowProbability()) { + List groupByClause = genGroupBys(innerQueryColumns, null); + innerQuery.setGroupByClause(groupByClause); + if (groupByClause.size() > 0 && Randomly.getBooleanWithRatherLowProbability()) { + DuckDBExpressionGenerator havingGen = new DuckDBExpressionGenerator(state).setColumns(innerQueryRandomTables.getColumns()); + DuckDBExpression havingExpr = havingGen.generateExpression(); + innerQuery.setHavingClause(havingExpr); + } + } + + this.foldedExpr = innerQuery; + + + // outer query + DuckDBSelect outerQuery = new DuckDBSelect(); + List tableRefList = outerQueryRandomTables.getTables().stream() + .map(t -> new DuckDBTableReference(t)).collect(Collectors.toList()); + outerQuery.setFromList(tableRefList); + tablesFromOuterContext = outerQueryRandomTables.getTables(); + + List outerQueryFetchColumns = new ArrayList<>(); + int columnIdx = 0; + for (DuckDBColumn c : outerQueryRandomTables.getColumns()) { + DuckDBColumnReference cRef = new DuckDBColumnReference(c); + String aliasName = "c" + String.valueOf(columnIdx); + DuckDBAlias columnAlias = new DuckDBAlias(cRef, aliasName); + outerQueryFetchColumns.add(columnAlias); + columnIdx++; + } + + // add the expression to fetch clause + String aliasName = "c" + String.valueOf(columnIdx); + DuckDBAlias columnAlias = new DuckDBAlias(innerQuery, aliasName); + outerQueryFetchColumns.add(columnAlias); + + outerQuery.setFetchColumns(outerQueryFetchColumns); + + originalQueryString = DuckDBToStringVisitor.asString(outerQuery); + + Map> queryRes = null; + try { + queryRes = getQueryResult(originalQueryString, state); + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(e.getMessage()); + } + } + if (queryRes.get("c0").size() == 0) { + throw new IgnoreMeException(); + } + + // save the result first + selectResult.clear(); + selectResult.putAll(queryRes); + + // get the constant corresponding to each row from results + List summary = queryRes.remove("c" + String.valueOf(columnIdx)); + + LinkedHashMap> dbstate = new LinkedHashMap<>(); + + for (int i = 0; i < outerQueryFetchColumns.size() - 1; ++i) { + // do not put the last fetch column to values + DuckDBAlias cAlias = (DuckDBAlias) outerQueryFetchColumns.get(i); + DuckDBColumnReference cRef = (DuckDBColumnReference) cAlias.getExpr(); + String columnName = cAlias.getAlias(); + dbstate.put(cRef, queryRes.get(columnName)); + } + this.constantResOfFoldedExpr = new DuckDBResultMap(dbstate, summary); + + return outerQuery; + } + + private DuckDBSelect genSimpleSelect() { + DuckDBTables randomTables = s.getRandomTableNonEmptyTables(); + List columns = randomTables.getColumns(); + gen = new DuckDBExpressionGenerator(state).setColumns(columns); + List tables = randomTables.getTables(); + tablesFromOuterContext = randomTables.getTables(); + List tableRefList = tables.stream() + .map(t -> new DuckDBTableReference(t)).collect(Collectors.toList()); + + DuckDBSelect select = new DuckDBSelect(); + + if (Randomly.getBoolean()) { + List joins = genJoinExpression(tableRefList, null); + select.setJoinList(joins); + this.joinsInExpr = joins.stream() + .map(expr -> (DuckDBJoin) expr) + .collect(Collectors.toList()); + } + + select.setFromList(tableRefList.stream().collect(Collectors.toList())); + + DuckDBExpression randomWhereCondition = gen.generateExpression(); + this.foldedExpr = randomWhereCondition; + + List fetchColumns = new ArrayList<>(); + int columnIdx = 0; + for (DuckDBColumn c : randomTables.getColumns()) { + DuckDBColumnReference cRef = new DuckDBColumnReference(c); + String aliasName = "c" + String.valueOf(columnIdx); + DuckDBAlias columnAlias = new DuckDBAlias(cRef, aliasName); + fetchColumns.add(columnAlias); + columnIdx++; + } + + // add the expression to fetch clause + String exprAliasName = "c" + String.valueOf(columnIdx); + DuckDBAlias exprAlias = new DuckDBAlias(randomWhereCondition, exprAliasName); + fetchColumns.add(exprAlias); + + select.setFetchColumns(fetchColumns); + + Map> queryRes = null; + try { + queryRes = getQueryResult(DuckDBToStringVisitor.asString(select), state); + } catch (SQLException e){ + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(e.getMessage()); + } + } + if (queryRes.get("c0").size() == 0) { + throw new IgnoreMeException(); + } + + // save the result first + selectResult.clear(); + selectResult.putAll(queryRes); + + // get the constant corresponding to each row from results + List summary = queryRes.remove(exprAliasName); + Map columnsType = getColumnTypeFromSelect(select); + DuckDBCompositeDataType exprType = columnsType.get(fetchColumns.size() -1 ); + List constantRes = new ArrayList<>(); + for (DuckDBExpression e : summary) { + DuckDBConstant c = (DuckDBConstant) e; + constantRes.add(new DuckDBConstantWithType(c, exprType)); + } + + LinkedHashMap> dbstate = new LinkedHashMap<>(); + + for (int i = 0; i < fetchColumns.size() - 1; ++i) { + // do not put the last fetch column to values + DuckDBAlias cAlias = (DuckDBAlias) fetchColumns.get(i); + DuckDBColumnReference cRef = (DuckDBColumnReference) cAlias.getExpr(); + String columnName = cAlias.getAlias(); + dbstate.put(cRef, queryRes.get(columnName)); + } + this.constantResOfFoldedExpr = new DuckDBResultMap(dbstate, constantRes); + + return select; + } + + private List genJoinExpression( + List tableList, DuckDBExpression specificCondition) { + List joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBooleanWithRatherLowProbability()) { + DuckDBTableReference leftTable = tableList.remove(0); + DuckDBTableReference rightTable = tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + DuckDBExpressionGenerator joinGen = new DuckDBExpressionGenerator(state).setColumns(columns); + DuckDBExpression onPredicate = joinGen.generateExpression(); + if (specificCondition != null && Randomly.getBooleanWithRatherLowProbability()) { + Operator operator = Randomly.fromList(Arrays.asList(DuckDBBinaryComparisonOperator.getRandom(), DuckDBBinaryArithmeticOperator.getRandom(), DuckDBBinaryLogicalOperator.getRandom())); + onPredicate = new DuckDBBinaryOperator(onPredicate, specificCondition, operator); + } + + switch (DuckDBJoin.JoinType.getRandom()) { + case INNER: + joinExpressions.add(DuckDBJoin.createInnerJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case NATURAL: + joinExpressions.add(DuckDBJoin.createNaturalJoin(leftTable, rightTable, OuterType.getRandom())); + break; + case LEFT: + joinExpressions + .add(DuckDBJoin.createLeftOuterJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + case RIGHT: + joinExpressions + .add(DuckDBJoin.createRightOuterJoin(leftTable, rightTable, joinGen.generateExpression())); + break; + default: + throw new AssertionError(); + } + } + return joinExpressions; + } + + private List genOrderBys(DuckDBExpression specificCondition) { + List expr = gen.generateExpressions(Randomly.smallNumber() + 1); + List newExpr = new ArrayList<>(expr.size()); + for (DuckDBExpression curExpr : expr) { + if (specificCondition != null && Randomly.getBooleanWithRatherLowProbability()) { + Operator operator = Randomly.fromList(Arrays.asList(DuckDBBinaryComparisonOperator.getRandom(), DuckDBBinaryArithmeticOperator.getRandom(), DuckDBBinaryLogicalOperator.getRandom())); + curExpr = new DuckDBBinaryOperator(curExpr, specificCondition, operator); + } + if (Randomly.getBoolean()) { + curExpr = new DuckDBOrderingTerm(curExpr, Ordering.getRandom()); + } + newExpr.add(curExpr); + } + return newExpr; + }; + + private List genGroupBys(List columns, DuckDBExpression specificCondition) { + DuckDBExpressionGenerator groupByGen = new DuckDBExpressionGenerator(state).setColumns(columns); + List exprs = groupByGen.generateGroupBys(); + List newExpr = new ArrayList<>(exprs.size()); + for (DuckDBExpression curExpr : exprs) { + if (specificCondition != null && Randomly.getBooleanWithRatherLowProbability()) { + Operator operator = Randomly.fromList(Arrays.asList(DuckDBBinaryComparisonOperator.getRandom(), DuckDBBinaryArithmeticOperator.getRandom(), DuckDBBinaryLogicalOperator.getRandom())); + curExpr = new DuckDBBinaryOperator(curExpr, specificCondition, operator); + } + newExpr.add(curExpr); + } + return newExpr; + } + + private Map> getQueryResult(String queryString, DuckDBGlobalState state) throws SQLException { + Map> result = new LinkedHashMap<>(); + if (options.logEachSelect()) { + logger.writeCurrent(queryString); + } + try (Statement s = this.con.createStatement()) { + try (ResultSet rs = s.executeQuery(queryString)) { + ResultSetMetaData metaData = rs.getMetaData(); + Integer columnCount = metaData.getColumnCount(); + Map idxNameMap = new HashMap<>(); + for (int i = 1; i <= columnCount; i++) { + result.put("c" + String.valueOf(i-1), new ArrayList<>()); + idxNameMap.put(i, "c" + String.valueOf(i-1)); + } + + int resultRows = 0; + while (rs.next()) { + for (int i = 1; i <= columnCount; i++) { + try { + Object value = rs.getObject(i); + DuckDBExpression constant; + if (rs.wasNull()) { + constant = DuckDBConstant.createNullConstant(); + } else if (value instanceof Integer) { + constant = DuckDBConstant.createIntConstant(BigInteger.valueOf((Integer) value)); + } else if (value instanceof Short) { + constant = DuckDBConstant.createIntConstant(BigInteger.valueOf((Short) value)); + } else if (value instanceof BigInteger) { + constant = DuckDBConstant.createIntConstant((BigInteger) value); + } else if (value instanceof Long) { + constant = DuckDBConstant.createIntConstant(BigInteger.valueOf((long) value)); + } else if (value instanceof Double) { + constant = DuckDBConstant.createFloatConstant((double) value); + } else if (value instanceof Float) { + constant = DuckDBConstant.createFloatConstant(Double.valueOf((Float)value)); + } else if (value instanceof Boolean) { + constant = DuckDBConstant.createBooleanConstant((Boolean) value); + }else if (value instanceof java.sql.Timestamp) { + constant = DuckDBConstant.createTimestampConstant(((java.sql.Timestamp) value).getTime()); + } else if (value instanceof java.sql.Date) { + constant = DuckDBConstant.createDateConstant(((java.sql.Date) value).getTime()); + } + else if (value instanceof String) { + constant = DuckDBConstant.createStringConstant((String) value); + } else { + throw new IgnoreMeException(); + } + List v = result.get(idxNameMap.get(i)); + v.add(constant); + } catch (Exception e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(e.getMessage()); + } + } + } + ++resultRows; + if (resultRows > 100) { + throw new IgnoreMeException(); + } + } + rs.close(); + Main.nrSuccessfulActions.addAndGet(1); + } catch (Exception e) { + Main.nrUnsuccessfulActions.addAndGet(1); + if (e.getMessage() == null || errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + state.getState().getLocalState().log(queryString); + throw new AssertionError(e.getMessage()); + } + } + } + return result; + } + + private DuckDBTable genTemporaryTable(DuckDBSelect select, String tableName) { + int columnNumber = select.getFetchColumns().size(); + Map idxTypeMap = this.getColumnTypeFromSelect(select); + + List databaseColumns = new ArrayList<>(); + for (int i = 0; i < columnNumber; ++i) { + String columnName = "c" + String.valueOf(i); + DuckDBColumn column = new DuckDBColumn(columnName, idxTypeMap.get(i), false, false); + databaseColumns.add(column); + } + DuckDBTable table = new DuckDBTable(tableName, databaseColumns, false); + for (DuckDBColumn c : databaseColumns) { + c.setTable(table); + } + + return table; + } + + private DuckDBTable createTemporaryTable(DuckDBSelect select, String tableName) throws SQLException { + String selectString = DuckDBToStringVisitor.asString(select); + Integer columnNumber = select.getFetchColumns().size(); + + Map idxTypeMap = this.getColumnTypeFromSelect(select); + + StringBuilder sb = new StringBuilder(); + sb.append("CREATE TABLE " + tableName + " ("); + for (int i = 0; i < columnNumber; ++i) { + String columnTypeName = idxTypeMap.get(i).toString(); + sb.append("c" + String.valueOf(i) + " " + columnTypeName + ", "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append(");"); + String crateTableString = sb.toString(); + if (options.logEachSelect()) { + logger.writeCurrent(crateTableString); + } + try (Statement s = this.con.createStatement()) { + try { + s.execute(crateTableString); + } catch (SQLException e) { + throw new IgnoreMeException(); + } + } + + StringBuilder sb2 = new StringBuilder(); + sb2.append("INSERT INTO " + tableName + " "+ selectString); + String insertValueString = sb2.toString(); + if (options.logEachSelect()) { + logger.writeCurrent(insertValueString); + } + try (Statement s = this.con.createStatement()) { + try { + s.execute(insertValueString); + } catch (SQLException e) { + throw new IgnoreMeException(); + } + } + + List databaseColumns = new ArrayList<>(); + for (int i = 0; i < columnNumber; ++i) { + String columnName = "c" + String.valueOf(i); + DuckDBColumn column = new DuckDBColumn(columnName, idxTypeMap.get(i), false, false); + databaseColumns.add(column); + } + DuckDBTable table = new DuckDBTable(tableName, databaseColumns, false); + for (DuckDBColumn c : databaseColumns) { + c.setTable(table); + } + + return table; + } + + private void dropTemporaryTable(String tableName) throws SQLException { + String dropString = "DROP TABLE " + tableName + ";"; + if (options.logEachSelect()) { + logger.writeCurrent(dropString); + } + try (Statement s = this.con.createStatement()) { + try { + s.execute(dropString); + } catch (SQLException e) { + throw new IgnoreMeException(); + } + } + } + + private Map getColumnTypeFromSelect(DuckDBSelect select) { + List fetchColumns = select.getFetchColumns(); + List newFetchColumns = new ArrayList<>(); + for(DuckDBExpression column : fetchColumns) { + newFetchColumns.add(column); + DuckDBAlias columnAlias = (DuckDBAlias) column; + DuckDBTypeofNode typeofColumn = new DuckDBTypeofNode(columnAlias.getExpr()); + newFetchColumns.add(typeofColumn); + } + DuckDBSelect newSelect = new DuckDBSelect(select); + newSelect.setFetchColumns(newFetchColumns); + Map> typeResult = null; + try { + typeResult = getQueryResult(DuckDBToStringVisitor.asString(newSelect), state); + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(e.getMessage()); + } + } + if (typeResult == null) { + throw new IgnoreMeException(); + } + Map idxTypeMap = new HashMap<>(); + for (int i = 0; i * 2 < typeResult.size(); ++i) { + String columnName = "c" + String.valueOf(i * 2 + 1); + DuckDBExpression t = typeResult.get(columnName).get(0); + DuckDBTextConstant tString = (DuckDBTextConstant) t; + String typeName = tString.getValue(); + DuckDBCompositeDataType cType = new DuckDBCompositeDataType(typeName); + idxTypeMap.put(i, cType); + } + return idxTypeMap; + } + + private boolean compareResult(Map> r1, Map> r2) { + if (r1.size() != r2.size()) { + return false; + } + for (Map.Entry < String, List > entry: r1.entrySet()) { + String currentKey = entry.getKey(); + if (!r2.containsKey(currentKey)) { + return false; + } + List v1= entry.getValue(); + List v2= r2.get(currentKey); + if (v1.size() != v2.size()) { + return false; + } + List v1Value = new ArrayList<>(v1.stream().map(c -> ((DuckDBConstant)c).toString()).collect(Collectors.toList())); + List v2Value = new ArrayList<>(v2.stream().map(c -> ((DuckDBConstant)c).toString()).collect(Collectors.toList())); + Collections.sort(v1Value); + Collections.sort(v2Value); + // if (!v1Value.equals(v2Value)) { + // return false; + // } + for (int i = 0; i < v1Value.size(); ++i) { + if (!v1Value.get(i).equals(v2Value.get(i))) { + String regx = "[+-]*\\d+\\.?\\d*[Ee]*[+-]*\\d+"; + Pattern pattern = Pattern.compile(regx); + if (pattern.matcher(v1Value.get(i)).matches() && pattern.matcher(v2Value.get(i)).matches()) { + if (!v1Value.get(i).substring(0, 6).equals(v2Value.get(i).substring(0, 6))) { + return false; + } + } else { + return false; + } + } + } + } + return true; + } + + public boolean useSubquery() { + if (this.state.getDbmsSpecificOptions().coddTestModel.equals("random")) { + return Randomly.getBoolean(); + } else if (this.state.getDbmsSpecificOptions().coddTestModel.equals("expression")) { + return false; + } else if (this.state.getDbmsSpecificOptions().coddTestModel.equals("subquery")) { + return true; + } else { + System.out.printf("Wrong option of --coddtest-model, should be one of: random, expression, subquery"); + System.exit(1); + return false; + } + } + + public boolean useCorrelatedSubquery() { + return Randomly.getBoolean(); + } + + public boolean testCommonTableExpression() { + return false; + } + public boolean testDerivedTable() { + return true; + } + public boolean testInsert() { + return false; + } + + @Override + public String getLastQueryString() { + return originalQueryString; + } + + @Override + public Reproducer getLastReproducer() { + return reproducer; + } +} From 5db6e3ca486865204c535c47dc0a2669ba1c8489 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Sat, 11 Jan 2025 10:46:02 +0800 Subject: [PATCH 04/16] enable CTE and insert test in CODDTest for DuckDB --- src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java b/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java index a102a9ee1..b0d3d5850 100644 --- a/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java +++ b/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java @@ -897,13 +897,13 @@ public boolean useCorrelatedSubquery() { } public boolean testCommonTableExpression() { - return false; + return true; } public boolean testDerivedTable() { return true; } public boolean testInsert() { - return false; + return true; } @Override From dbd8c8933634c460c47600f6a94f66e530aa17ad Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Sun, 12 Jan 2025 20:26:23 +0800 Subject: [PATCH 05/16] support CODDTest for CockroachDB --- .../cockroachdb/CockroachDBOptions.java | 3 + .../cockroachdb/CockroachDBOracleFactory.java | 8 + .../cockroachdb/CockroachDBSchema.java | 52 + .../CockroachDBToStringVisitor.java | 169 +++ .../cockroachdb/CockroachDBVisitor.java | 42 + .../cockroachdb/ast/CockroachDBAlias.java | 21 +- .../ast/CockroachDBAllOperator.java | 26 + .../ast/CockroachDBAnyOperator.java | 26 + .../cockroachdb/ast/CockroachDBConstant.java | 87 ++ .../cockroachdb/ast/CockroachDBExists.java | 24 + .../ast/CockroachDBExpressionBag.java | 18 + .../cockroachdb/ast/CockroachDBResultMap.java | 33 + .../cockroachdb/ast/CockroachDBSelect.java | 6 + .../CockroachDBTableAndColumnReference.java | 15 + .../cockroachdb/ast/CockroachDBTypeof.java | 11 + .../cockroachdb/ast/CockroachDBValues.java | 19 + .../ast/CockroachDBWithClasure.java | 24 + .../oracle/CockroachDBCODDTestOracle.java | 1275 +++++++++++++++++ 18 files changed, 1841 insertions(+), 18 deletions(-) create mode 100644 src/sqlancer/cockroachdb/ast/CockroachDBAllOperator.java create mode 100644 src/sqlancer/cockroachdb/ast/CockroachDBAnyOperator.java create mode 100644 src/sqlancer/cockroachdb/ast/CockroachDBExists.java create mode 100644 src/sqlancer/cockroachdb/ast/CockroachDBExpressionBag.java create mode 100644 src/sqlancer/cockroachdb/ast/CockroachDBResultMap.java create mode 100644 src/sqlancer/cockroachdb/ast/CockroachDBTableAndColumnReference.java create mode 100644 src/sqlancer/cockroachdb/ast/CockroachDBTypeof.java create mode 100644 src/sqlancer/cockroachdb/ast/CockroachDBValues.java create mode 100644 src/sqlancer/cockroachdb/ast/CockroachDBWithClasure.java create mode 100644 src/sqlancer/cockroachdb/oracle/CockroachDBCODDTestOracle.java diff --git a/src/sqlancer/cockroachdb/CockroachDBOptions.java b/src/sqlancer/cockroachdb/CockroachDBOptions.java index ce8a207d6..abaaa8d08 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOptions.java +++ b/src/sqlancer/cockroachdb/CockroachDBOptions.java @@ -30,6 +30,9 @@ public class CockroachDBOptions implements DBMSSpecificOptions getTestOracleFactory() { return Arrays.asList(oracle); diff --git a/src/sqlancer/cockroachdb/CockroachDBOracleFactory.java b/src/sqlancer/cockroachdb/CockroachDBOracleFactory.java index 96fbc22ce..0b97bc29b 100644 --- a/src/sqlancer/cockroachdb/CockroachDBOracleFactory.java +++ b/src/sqlancer/cockroachdb/CockroachDBOracleFactory.java @@ -7,7 +7,9 @@ import sqlancer.IgnoreMeException; import sqlancer.OracleFactory; +import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; +import sqlancer.cockroachdb.oracle.CockroachDBCODDTestOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPAggregateOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPDistinctOracle; import sqlancer.cockroachdb.oracle.tlp.CockroachDBTLPExtendedWhereOracle; @@ -133,6 +135,12 @@ public TestOracle create( public boolean requiresAllTablesToContainRows() { return true; } + }, + CODDTest { + @Override + public TestOracle create(CockroachDBGlobalState globalState) throws SQLException { + return new CockroachDBCODDTestOracle(globalState); + } }; } diff --git a/src/sqlancer/cockroachdb/CockroachDBSchema.java b/src/sqlancer/cockroachdb/CockroachDBSchema.java index 708afef92..462a07b65 100644 --- a/src/sqlancer/cockroachdb/CockroachDBSchema.java +++ b/src/sqlancer/cockroachdb/CockroachDBSchema.java @@ -41,6 +41,8 @@ public static class CockroachDBCompositeDataType { private CockroachDBCompositeDataType elementType; + private String typeName = ""; + public CockroachDBCompositeDataType(CockroachDBDataType dataType) { this.dataType = dataType; this.size = -1; @@ -60,6 +62,53 @@ public CockroachDBCompositeDataType(CockroachDBDataType dataType, CockroachDBCom this.elementType = elementType; } + + public CockroachDBCompositeDataType(String typeName) { + switch (typeName) { + case "smallint": + case "integer": + case "bigint": + this.dataType = CockroachDBDataType.INT; + break; + case "boolean": + this.dataType = CockroachDBDataType.BOOL; + break; + case "text": + case "character": + case "character varying": + case "name": + case "regclass": + this.dataType = CockroachDBDataType.STRING; + break; + case "numeric": + this.dataType = CockroachDBDataType.DECIMAL; + break; + case "double precision": + case "real": + this.dataType = CockroachDBDataType.FLOAT; + break; + case "bit": + case "bit varying": + this.dataType = CockroachDBDataType.BIT; + break; + + // for ire, maybe not correct + case "unknown": + this.dataType = CockroachDBDataType.BOOL; + break; + default: + throw new AssertionError(typeName); + } + + // TODO: check later + if (typeName.equals("bit varying")) { + this.size = 4; + } else { + this.size = -1; + } + this.typeName = typeName; + } + public CockroachDBDataType getPrimitiveDataType() { return dataType; } @@ -85,6 +134,9 @@ public static CockroachDBCompositeDataType getBit(int size) { @Override public String toString() { + if (this.typeName != "") { + return this.typeName; + } switch (dataType) { case INT: switch (size) { diff --git a/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java b/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java index 67abfdbbe..47e672731 100644 --- a/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java +++ b/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java @@ -1,20 +1,35 @@ package sqlancer.cockroachdb; +import java.util.LinkedHashMap; +import java.util.List; import java.util.stream.Collectors; import sqlancer.Randomly; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; import sqlancer.cockroachdb.ast.CockroachDBAggregate; +import sqlancer.cockroachdb.ast.CockroachDBAlias; +import sqlancer.cockroachdb.ast.CockroachDBAllOperator; +import sqlancer.cockroachdb.ast.CockroachDBAnyOperator; import sqlancer.cockroachdb.ast.CockroachDBBetweenOperation; import sqlancer.cockroachdb.ast.CockroachDBCaseOperation; import sqlancer.cockroachdb.ast.CockroachDBColumnReference; import sqlancer.cockroachdb.ast.CockroachDBConstant; +import sqlancer.cockroachdb.ast.CockroachDBConstant.CockroachDBNullConstant; +import sqlancer.cockroachdb.ast.CockroachDBExists; import sqlancer.cockroachdb.ast.CockroachDBExpression; +import sqlancer.cockroachdb.ast.CockroachDBExpressionBag; import sqlancer.cockroachdb.ast.CockroachDBFunctionCall; import sqlancer.cockroachdb.ast.CockroachDBInOperation; import sqlancer.cockroachdb.ast.CockroachDBJoin; import sqlancer.cockroachdb.ast.CockroachDBMultiValuedComparison; +import sqlancer.cockroachdb.ast.CockroachDBResultMap; import sqlancer.cockroachdb.ast.CockroachDBSelect; +import sqlancer.cockroachdb.ast.CockroachDBTableAndColumnReference; import sqlancer.cockroachdb.ast.CockroachDBTableReference; +import sqlancer.cockroachdb.ast.CockroachDBTypeof; +import sqlancer.cockroachdb.ast.CockroachDBValues; +import sqlancer.cockroachdb.ast.CockroachDBWithClasure; import sqlancer.common.visitor.ToStringVisitor; public class CockroachDBToStringVisitor extends ToStringVisitor implements CockroachDBVisitor { @@ -75,6 +90,9 @@ public void visit(CockroachDBBetweenOperation op) { @Override public void visit(CockroachDBSelect select) { + if (select.getWithClause() != null) { + visit(select.getWithClause()); + } sb.append("SELECT "); if (select.isDistinct()) { sb.append("DISTINCT "); @@ -231,4 +249,155 @@ public void visit(CockroachDBMultiValuedComparison comp) { sb.append(")"); sb.append(")"); } + + + @Override + public void visit(CockroachDBExists existsExpr) { + if (existsExpr.getNegated()) { + sb.append(" NOT"); + } + sb.append(" EXISTS("); + visit(existsExpr.getExpression()); + sb.append(")"); + } + + @Override + public void visit(CockroachDBExpressionBag exprBag) { + visit(exprBag.getInnerExpr()); + } + + @Override + public void visit(CockroachDBValues values) { + LinkedHashMap> vs = values.getValues(); + int size = vs.values().iterator().next().size(); + sb.append("(VALUES "); + for (int i = 0; i < size; i++) { + sb.append("("); + for (CockroachDBColumn c: vs.keySet()) { + sb.append(vs.get(c).get(i).toString()); + if (!(vs.get(c).get(i) instanceof CockroachDBNullConstant)) { + if (c.getType() != null) { + sb.append("::" + c.getType().toString()); + } + } + sb.append(", "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append("), "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append(")"); + } + + @Override + public void visit(CockroachDBWithClasure withClasure) { + sb.append("WITH "); + visit(withClasure.getLeft()); + sb.append(" AS ("); + visit(withClasure.getRight()); + sb.append(") "); + } + + @Override + public void visit(CockroachDBTableAndColumnReference tableAndColumnReference) { + CockroachDBTable table = tableAndColumnReference.getTable(); + sb.append(table.getName()); + sb.append("("); + sb.append(table.getColumnsAsString()); + sb.append(") "); + } + + @Override + public void visit(CockroachDBAlias alias) { + CockroachDBExpression e = alias.getExpression(); + if (e instanceof CockroachDBSelect) { + sb.append("("); + } + visit(e); + if (e instanceof CockroachDBSelect) { + sb.append(")"); + } + sb.append(" AS "); + sb.append(alias.getAlias()); + } + + @Override + public void visit(CockroachDBTypeof typeOf) { + sb.append("pg_typeof("); + visit(typeOf.getExpr()); + sb.append(")"); + } + + @Override + public void visit(CockroachDBResultMap expr) { + // use CASE WHEN to express the constant result of a expression + LinkedHashMap> dbstate = expr.getDbStates(); + List result = expr.getResult(); + int size = dbstate.values().iterator().next().size(); + if (size == 0) { + sb.append(" NULL "); + return; + } + sb.append(" CASE "); + for (int i = 0; i < size; i++) { + sb.append("WHEN "); + for (CockroachDBColumnReference columnRef : dbstate.keySet()) { + visit(columnRef); + if (dbstate.get(columnRef).get(i) instanceof CockroachDBNullConstant) { + sb.append(" IS NULL"); + } else { + sb.append(" = "); + visit(dbstate.get(columnRef).get(i)); + if (!(dbstate.get(columnRef).get(i) instanceof CockroachDBNullConstant)) { + if (columnRef.getColumn().getType() != null) { + sb.append("::" + columnRef.getColumn().getType().toString()); + } + } + } + sb.append(" AND "); + } + for (int k = 0; k < 4; k++) { + sb.deleteCharAt(sb.length() - 1); + } + sb.append("THEN "); + visit(result.get(i)); + if (!(result.get(i) instanceof CockroachDBNullConstant)) { + if (expr.getResultType() != null) { + sb.append("::" + expr.getResultType().toString()); + } + } + sb.append(" "); + } + sb.append("END "); + } + + @Override + public void visit(CockroachDBAllOperator allOperation) { + sb.append("("); + visit(allOperation.getLeftExpr()); + sb.append(") "); + sb.append(allOperation.getOperator()); + sb.append(" ALL ("); + // if (allOperation.getRightExpr() instanceof CockroachDBValues) { + // sb.append("SELECT "); + // } + visit(allOperation.getRightExpr()); + sb.append(")"); + } + + @Override + public void visit(CockroachDBAnyOperator anyOperation) { + sb.append("("); + visit(anyOperation.getLeftExpr()); + sb.append(") "); + sb.append(anyOperation.getOperator()); + sb.append(" ANY ("); + // if (anyOperation.getRightExpr() instanceof CockroachDBValues) { + // sb.append("SELECT "); + // } + visit(anyOperation.getRightExpr()); + sb.append(")"); + } } diff --git a/src/sqlancer/cockroachdb/CockroachDBVisitor.java b/src/sqlancer/cockroachdb/CockroachDBVisitor.java index 01b434607..d82c6c948 100644 --- a/src/sqlancer/cockroachdb/CockroachDBVisitor.java +++ b/src/sqlancer/cockroachdb/CockroachDBVisitor.java @@ -1,17 +1,27 @@ package sqlancer.cockroachdb; import sqlancer.cockroachdb.ast.CockroachDBAggregate; +import sqlancer.cockroachdb.ast.CockroachDBAlias; +import sqlancer.cockroachdb.ast.CockroachDBAllOperator; +import sqlancer.cockroachdb.ast.CockroachDBAnyOperator; import sqlancer.cockroachdb.ast.CockroachDBBetweenOperation; import sqlancer.cockroachdb.ast.CockroachDBCaseOperation; import sqlancer.cockroachdb.ast.CockroachDBColumnReference; import sqlancer.cockroachdb.ast.CockroachDBConstant; +import sqlancer.cockroachdb.ast.CockroachDBExists; import sqlancer.cockroachdb.ast.CockroachDBExpression; +import sqlancer.cockroachdb.ast.CockroachDBExpressionBag; import sqlancer.cockroachdb.ast.CockroachDBFunctionCall; import sqlancer.cockroachdb.ast.CockroachDBInOperation; import sqlancer.cockroachdb.ast.CockroachDBJoin; import sqlancer.cockroachdb.ast.CockroachDBMultiValuedComparison; +import sqlancer.cockroachdb.ast.CockroachDBResultMap; import sqlancer.cockroachdb.ast.CockroachDBSelect; +import sqlancer.cockroachdb.ast.CockroachDBTableAndColumnReference; import sqlancer.cockroachdb.ast.CockroachDBTableReference; +import sqlancer.cockroachdb.ast.CockroachDBTypeof; +import sqlancer.cockroachdb.ast.CockroachDBValues; +import sqlancer.cockroachdb.ast.CockroachDBWithClasure; public interface CockroachDBVisitor { @@ -37,6 +47,18 @@ public interface CockroachDBVisitor { void visit(CockroachDBMultiValuedComparison comp); + // CODDTest + void visit(CockroachDBExists existsExpr); + void visit(CockroachDBExpressionBag exprBag); + void visit(CockroachDBValues values); + void visit(CockroachDBWithClasure withClasure); + void visit(CockroachDBTableAndColumnReference tableAndColumnReference); + void visit(CockroachDBAlias alias); + void visit(CockroachDBTypeof typeOf); + void visit(CockroachDBResultMap resMap); + void visit(CockroachDBAllOperator allOperator); + void visit(CockroachDBAnyOperator anyOperator); + default void visit(CockroachDBExpression expr) { if (expr instanceof CockroachDBConstant) { visit((CockroachDBConstant) expr); @@ -60,6 +82,26 @@ default void visit(CockroachDBExpression expr) { visit((CockroachDBAggregate) expr); } else if (expr instanceof CockroachDBMultiValuedComparison) { visit((CockroachDBMultiValuedComparison) expr); + } else if (expr instanceof CockroachDBExists) { + visit((CockroachDBExists) expr); + } else if (expr instanceof CockroachDBExpressionBag) { + visit((CockroachDBExpressionBag) expr); + } else if (expr instanceof CockroachDBValues) { + visit((CockroachDBValues) expr); + } else if (expr instanceof CockroachDBWithClasure) { + visit ((CockroachDBWithClasure) expr); + } else if (expr instanceof CockroachDBTableAndColumnReference) { + visit ((CockroachDBTableAndColumnReference) expr); + } else if (expr instanceof CockroachDBAlias) { + visit ((CockroachDBAlias) expr); + } else if (expr instanceof CockroachDBTypeof) { + visit ((CockroachDBTypeof) expr); + } else if (expr instanceof CockroachDBResultMap) { + visit ((CockroachDBResultMap) expr); + } else if (expr instanceof CockroachDBAllOperator) { + visit((CockroachDBAllOperator) expr); + } else if (expr instanceof CockroachDBAnyOperator) { + visit((CockroachDBAnyOperator) expr); } else { throw new AssertionError(expr.getClass()); } diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBAlias.java b/src/sqlancer/cockroachdb/ast/CockroachDBAlias.java index 3f083b5e3..1a879e591 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBAlias.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBAlias.java @@ -1,8 +1,6 @@ package sqlancer.cockroachdb.ast; -import sqlancer.common.visitor.UnaryOperation; - -public class CockroachDBAlias implements UnaryOperation, CockroachDBExpression { +public class CockroachDBAlias implements CockroachDBExpression { private final CockroachDBExpression expr; private final String alias; @@ -12,24 +10,11 @@ public CockroachDBAlias(CockroachDBExpression expr, String alias) { this.alias = alias; } - @Override public CockroachDBExpression getExpression() { return expr; } - @Override - public String getOperatorRepresentation() { - return " as " + alias; - } - - @Override - public OperatorKind getOperatorKind() { - return OperatorKind.POSTFIX; + public String getAlias() { + return alias; } - - @Override - public boolean omitBracketsWhenPrinting() { - return true; - } - } diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBAllOperator.java b/src/sqlancer/cockroachdb/ast/CockroachDBAllOperator.java new file mode 100644 index 000000000..721067c0a --- /dev/null +++ b/src/sqlancer/cockroachdb/ast/CockroachDBAllOperator.java @@ -0,0 +1,26 @@ +package sqlancer.cockroachdb.ast; + +import sqlancer.cockroachdb.ast.CockroachDBBinaryComparisonOperator.CockroachDBComparisonOperator; + +public class CockroachDBAllOperator implements CockroachDBExpression { + private final CockroachDBExpression leftExpr; + private final CockroachDBExpression rightExpr; + private final CockroachDBComparisonOperator op; + + + public CockroachDBAllOperator(CockroachDBExpression leftExpr, CockroachDBExpression rightExpr, CockroachDBComparisonOperator op) { + this.leftExpr = leftExpr; + this.rightExpr = rightExpr; + this.op = op; + } + + public CockroachDBExpression getLeftExpr() { + return leftExpr; + } + public CockroachDBExpression getRightExpr() { + return rightExpr; + } + public String getOperator() { + return op.getTextRepresentation(); + } +} diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBAnyOperator.java b/src/sqlancer/cockroachdb/ast/CockroachDBAnyOperator.java new file mode 100644 index 000000000..e80aa8920 --- /dev/null +++ b/src/sqlancer/cockroachdb/ast/CockroachDBAnyOperator.java @@ -0,0 +1,26 @@ +package sqlancer.cockroachdb.ast; + +import sqlancer.cockroachdb.ast.CockroachDBBinaryComparisonOperator.CockroachDBComparisonOperator; + +public class CockroachDBAnyOperator implements CockroachDBExpression { + private final CockroachDBExpression leftExpr; + private final CockroachDBExpression rightExpr; + private final CockroachDBComparisonOperator op; + + + public CockroachDBAnyOperator(CockroachDBExpression leftExpr, CockroachDBExpression rightExpr, CockroachDBComparisonOperator op) { + this.leftExpr = leftExpr; + this.rightExpr = rightExpr; + this.op = op; + } + + public CockroachDBExpression getLeftExpr() { + return leftExpr; + } + public CockroachDBExpression getRightExpr() { + return rightExpr; + } + public String getOperator() { + return op.getTextRepresentation(); + } +} diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBConstant.java b/src/sqlancer/cockroachdb/ast/CockroachDBConstant.java index aa4dabcc4..d14e3e810 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBConstant.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBConstant.java @@ -1,5 +1,6 @@ package sqlancer.cockroachdb.ast; +import java.math.BigDecimal; import java.sql.Timestamp; import java.text.SimpleDateFormat; import java.util.List; @@ -12,6 +13,11 @@ public class CockroachDBConstant implements CockroachDBExpression { private CockroachDBConstant() { } + // only for CODDTest. in comparison, the float value always has different precision + public String toStringForComparison() { + return toString(); + } + public static class CockroachDBNullConstant extends CockroachDBConstant { @Override @@ -38,6 +44,53 @@ public long getValue() { return value; } + @Override + public String toStringForComparison() { + BigDecimal decimal = new BigDecimal(value); + return decimal.toPlainString(); + } + + } + + public static class CockroachDBBigDecimalConstant extends CockroachDBConstant { + + private final BigDecimal val; + + public CockroachDBBigDecimalConstant(BigDecimal val) { + this.val = val; + } + + public BigDecimal getValue() { + return this.val; + } + + @Override + public String toString() { + return String.valueOf(val); + } + + @Override + public String toStringForComparison() { + String valString = val.toPlainString(); + if (valString.contains(".")){ + if (valString.indexOf(".") > 7) { + return valString.substring(0, valString.indexOf(".")); + } + if (valString.length() < 5) { + if (valString.equals("0.0")) { + return "0"; + } + else + return valString; + } + else { + return valString.substring(0, 5); + } + } + else { + return valString; + } + } } public static class CockroachDBDoubleConstant extends CockroachDBConstant { @@ -58,10 +111,39 @@ public String toString() { return "FLOAT '+Inf'"; } else if (value == Double.NEGATIVE_INFINITY) { return "FLOAT '-Inf'"; + } else if (Double.isNaN(value)) { + return "FLOAT 'NaN'"; } return String.valueOf(value); } + @Override + public String toStringForComparison() { + if (value == Double.POSITIVE_INFINITY) { + return "FLOAT '+Inf'"; + } else if (value == Double.NEGATIVE_INFINITY) { + return "FLOAT '-Inf'"; + } else if (Double.isNaN(value)) { + return "Nan"; + } + BigDecimal decimal = new BigDecimal(value); + String stringDoubleValue = decimal.toPlainString(); + if (stringDoubleValue.contains(".") && stringDoubleValue.indexOf(".") > 7){ + return stringDoubleValue.substring(0, stringDoubleValue.indexOf(".")); + } + if (!stringDoubleValue.contains(".")) { + return stringDoubleValue; + } + if (stringDoubleValue.equals("0.0")) { + return "0"; + } + if (stringDoubleValue.length() < 5) { + return stringDoubleValue; + } + else { + return stringDoubleValue.substring(0, 5); + } + } } public static class CockroachDBTextConstant extends CockroachDBConstant { @@ -209,6 +291,11 @@ public static CockroachDBIntConstant createIntConstant(long val) { return new CockroachDBIntConstant(val); } + + public static CockroachDBBigDecimalConstant createDecimalConstant(BigDecimal val) { + return new CockroachDBBigDecimalConstant(val); + } + public static CockroachDBNullConstant createNullConstant() { return new CockroachDBNullConstant(); } diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBExists.java b/src/sqlancer/cockroachdb/ast/CockroachDBExists.java new file mode 100644 index 000000000..0db89cf9f --- /dev/null +++ b/src/sqlancer/cockroachdb/ast/CockroachDBExists.java @@ -0,0 +1,24 @@ +package sqlancer.cockroachdb.ast; + +public class CockroachDBExists implements CockroachDBExpression { + + private final CockroachDBExpression select; + private boolean negated = false; + + public CockroachDBExists(CockroachDBExpression select, boolean negated) { + this.select = select; + this.negated = negated; + } + + public void setNegated(boolean negated) { + this.negated = negated; + } + + public boolean getNegated() { + return this.negated; + } + + public CockroachDBExpression getExpression() { + return select; + } +} diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBExpressionBag.java b/src/sqlancer/cockroachdb/ast/CockroachDBExpressionBag.java new file mode 100644 index 000000000..1289f2b00 --- /dev/null +++ b/src/sqlancer/cockroachdb/ast/CockroachDBExpressionBag.java @@ -0,0 +1,18 @@ +package sqlancer.cockroachdb.ast; + +public class CockroachDBExpressionBag implements CockroachDBExpression { + private CockroachDBExpression innerExpr; + + public CockroachDBExpressionBag(CockroachDBExpression innerExpr) { + this.innerExpr = innerExpr; + } + + public void updateInnerExpr(CockroachDBExpression innerExpr) { + this.innerExpr = innerExpr; + } + + public CockroachDBExpression getInnerExpr() { + return innerExpr; + } + +} diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBResultMap.java b/src/sqlancer/cockroachdb/ast/CockroachDBResultMap.java new file mode 100644 index 000000000..4504d993e --- /dev/null +++ b/src/sqlancer/cockroachdb/ast/CockroachDBResultMap.java @@ -0,0 +1,33 @@ +package sqlancer.cockroachdb.ast; + +import java.util.LinkedHashMap; +import java.util.List; + +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBCompositeDataType; + +public class CockroachDBResultMap implements CockroachDBExpression { + private final LinkedHashMap> DBStates; + private final List results; + CockroachDBCompositeDataType resultType; + + public CockroachDBResultMap(LinkedHashMap> s, List r, CockroachDBCompositeDataType rt) { + this.DBStates = s; + this.results = r; + this.resultType = rt; + if (s.get(s.keySet().iterator().next()).size() != r.size()) { + throw new AssertionError(); + } + } + + public LinkedHashMap> getDbStates() { + return this.DBStates; + } + + public List getResult() { + return this.results; + } + + public CockroachDBCompositeDataType getResultType() { + return this.resultType; + } +} diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBSelect.java b/src/sqlancer/cockroachdb/ast/CockroachDBSelect.java index c043a9207..005053ea1 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBSelect.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBSelect.java @@ -14,6 +14,12 @@ public class CockroachDBSelect extends SelectBase impleme private boolean isDistinct; + public CockroachDBSelect() {} + public CockroachDBSelect(CockroachDBSelect other) { + super(other); + this.isDistinct = other.isDistinct; + } + public boolean isDistinct() { return isDistinct; } diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBTableAndColumnReference.java b/src/sqlancer/cockroachdb/ast/CockroachDBTableAndColumnReference.java new file mode 100644 index 000000000..bfcea9ec0 --- /dev/null +++ b/src/sqlancer/cockroachdb/ast/CockroachDBTableAndColumnReference.java @@ -0,0 +1,15 @@ +package sqlancer.cockroachdb.ast; + +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; + +public class CockroachDBTableAndColumnReference implements CockroachDBExpression { + private final CockroachDBTable table; + + public CockroachDBTableAndColumnReference(CockroachDBTable table) { + this.table = table; + } + + public CockroachDBTable getTable() { + return table; + } +} diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBTypeof.java b/src/sqlancer/cockroachdb/ast/CockroachDBTypeof.java new file mode 100644 index 000000000..88ba3a8a3 --- /dev/null +++ b/src/sqlancer/cockroachdb/ast/CockroachDBTypeof.java @@ -0,0 +1,11 @@ +package sqlancer.cockroachdb.ast; + +public class CockroachDBTypeof implements CockroachDBExpression { + private final CockroachDBExpression expr; + public CockroachDBTypeof(CockroachDBExpression expr) { + this.expr = expr; + } + public CockroachDBExpression getExpr() { + return expr; + } +} diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBValues.java b/src/sqlancer/cockroachdb/ast/CockroachDBValues.java new file mode 100644 index 000000000..bc97afb02 --- /dev/null +++ b/src/sqlancer/cockroachdb/ast/CockroachDBValues.java @@ -0,0 +1,19 @@ +package sqlancer.cockroachdb.ast; + +import java.util.LinkedHashMap; +import java.util.List; + +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; + +public class CockroachDBValues implements CockroachDBExpression { + + private final LinkedHashMap> values; + + public CockroachDBValues(LinkedHashMap> v) { + this.values = v; + } + + public LinkedHashMap> getValues() { + return this.values; + } +} diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBWithClasure.java b/src/sqlancer/cockroachdb/ast/CockroachDBWithClasure.java new file mode 100644 index 000000000..e33a22935 --- /dev/null +++ b/src/sqlancer/cockroachdb/ast/CockroachDBWithClasure.java @@ -0,0 +1,24 @@ +package sqlancer.cockroachdb.ast; + +public class CockroachDBWithClasure implements CockroachDBExpression { + + private CockroachDBExpression left; + private CockroachDBExpression right; + + public CockroachDBWithClasure(CockroachDBExpression left, CockroachDBExpression right) { + this.left = left; + this.right = right; + } + + public CockroachDBExpression getLeft() { + return this.left; + } + + public CockroachDBExpression getRight() { + return this.right; + } + + public void updateRight(CockroachDBExpression right) { + this.right = right; + } +} diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBCODDTestOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBCODDTestOracle.java new file mode 100644 index 000000000..fc87d79cc --- /dev/null +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBCODDTestOracle.java @@ -0,0 +1,1275 @@ +package sqlancer.cockroachdb.oracle; + +import java.math.BigDecimal; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.sql.Time; +import java.sql.Timestamp; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import sqlancer.IgnoreMeException; +import sqlancer.Main; +import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.cockroachdb.CockroachDBErrors; +import sqlancer.cockroachdb.CockroachDBSchema; +import sqlancer.cockroachdb.CockroachDBVisitor; +import sqlancer.cockroachdb.CockroachDBProvider.CockroachDBGlobalState; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBColumn; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBCompositeDataType; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBDataType; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTable; +import sqlancer.cockroachdb.CockroachDBSchema.CockroachDBTables; +import sqlancer.cockroachdb.ast.CockroachDBAggregate; +import sqlancer.cockroachdb.ast.CockroachDBAlias; +import sqlancer.cockroachdb.ast.CockroachDBAllOperator; +import sqlancer.cockroachdb.ast.CockroachDBAnyOperator; +import sqlancer.cockroachdb.ast.CockroachDBBinaryComparisonOperator; +import sqlancer.cockroachdb.ast.CockroachDBBinaryLogicalOperation; +import sqlancer.cockroachdb.ast.CockroachDBColumnReference; +import sqlancer.cockroachdb.ast.CockroachDBConstant; +import sqlancer.cockroachdb.ast.CockroachDBExists; +import sqlancer.cockroachdb.ast.CockroachDBExpression; +import sqlancer.cockroachdb.ast.CockroachDBExpressionBag; +import sqlancer.cockroachdb.ast.CockroachDBInOperation; +import sqlancer.cockroachdb.ast.CockroachDBJoin; +import sqlancer.cockroachdb.ast.CockroachDBOrderingTerm; +import sqlancer.cockroachdb.ast.CockroachDBResultMap; +import sqlancer.cockroachdb.ast.CockroachDBSelect; +import sqlancer.cockroachdb.ast.CockroachDBTableAndColumnReference; +import sqlancer.cockroachdb.ast.CockroachDBTableReference; +import sqlancer.cockroachdb.ast.CockroachDBTypeof; +import sqlancer.cockroachdb.ast.CockroachDBValues; +import sqlancer.cockroachdb.ast.CockroachDBWithClasure; +import sqlancer.cockroachdb.ast.CockroachDBAggregate.CockroachDBAggregateFunction; +import sqlancer.cockroachdb.ast.CockroachDBBinaryComparisonOperator.CockroachDBComparisonOperator; +import sqlancer.cockroachdb.ast.CockroachDBBinaryLogicalOperation.CockroachDBBinaryLogicalOperator; +import sqlancer.cockroachdb.ast.CockroachDBConstant.CockroachDBTextConstant; +import sqlancer.cockroachdb.gen.CockroachDBExpressionGenerator; +import sqlancer.common.oracle.CODDTestBase; +import sqlancer.common.oracle.TestOracle; + + +public class CockroachDBCODDTestOracle extends CODDTestBase implements TestOracle { + + private final CockroachDBSchema s; + private CockroachDBExpressionGenerator gen; + private Reproducer reproducer; + + private String tempTableName = "temp_table"; + + private CockroachDBExpression foldedExpr; + private CockroachDBExpression constantResOfFoldedExpr; + + private List tablesFromOuterContext = new ArrayList<>(); + private List joinsInExpr = null; + + Map> auxiliaryQueryResult = new HashMap<>(); + Map> selectResult = new HashMap<>(); + + Boolean useSubqueryAsFoldedExpr; + Boolean useCorrelatedSubqueryAsFoldedExpr; + + CockroachDBCompositeDataType foldedExpressionReturnType = null; + + public CockroachDBCODDTestOracle(CockroachDBGlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + CockroachDBErrors.addExpressionErrors(errors); + CockroachDBErrors.addTransactionErrors(errors); + errors.add("unable to vectorize execution plan"); + errors.add("mismatched physical types at index"); + errors.add("unknown signature: "); + errors.add("null rejection requested on non-null column"); + errors.add("computed column expression cannot reference computed columns"); + + // an error that hard to avoid, because use specific condition in JOIN/ORDER BY/GROUP BY/HAVING + errors.add("ERROR: no data source matches prefix:"); + + errors.add("expected computed column expression to have type bytes"); + errors.add("ERROR: hash-fingerprint: bytea encoded value ends with escape character"); + errors.add("ERROR: internal error: cannot overwrite"); + errors.add("volatile functions are not allowed in computed column"); + errors.add("the following columns are not indexable due to their type"); + errors.add("found in depended-on-by references, no such index in this relation"); + errors.add("context-dependent operators are not allowed in STORED COMPUTED COLUMN"); + errors.add("invalid value for kv.range_descriptor_cache.size"); + errors.add("could not parse JSON: unable to decode JSON"); + errors.add("cannot cast jsonb object to type bool"); + errors.add("ilike_escape(): invalid encoding of the first character in string"); + errors.add("ilike_escape(): invalid encoding of the last character in string"); + errors.add("ERROR: incompatible type annotation for"); + errors.add("ERROR: VALUES types float and decimal cannot be matched"); + errors.add("ERROR: VALUES types decimal and int cannot be matched"); + errors.add("ERROR: VALUES types decimal and float cannot be matched"); + errors.add("ERROR: VALUES types int and decimal cannot be matched"); + errors.add("ERROR: could not decorrelate subquery"); + errors.add("ERROR: unimplemented: apply joins with subqueries in the \"inner\" and \"outer\" contexts are not supported"); + errors.add("language: tag is not well-formed"); // a bug already reported + errors.add("expected subquery to be lazily planned as a routine"); // a bug already reported + errors.add("invalid encoding of the first character in string"); + errors.add("invalid encoding of the last character in string"); + + errors.add("value type string doesn't match type timetz of column"); + + // ERROR: subquery uses ungrouped column "rowid" from outer query + errors.add("ERROR: subquery uses ungrouped column "); + } + + @Override + public void check() throws Exception { + reproducer = null; + + joinsInExpr = null; + tablesFromOuterContext.clear(); + + useSubqueryAsFoldedExpr = useSubquery(); + useCorrelatedSubqueryAsFoldedExpr = useCorrelatedSubquery(); + + CockroachDBSelect auxiliaryQuery = null; + + if (useSubqueryAsFoldedExpr) { + if (useCorrelatedSubqueryAsFoldedExpr) { + auxiliaryQuery = genSelectWithCorrelatedSubquery(); + auxiliaryQueryString = CockroachDBVisitor.asString(auxiliaryQuery); + auxiliaryQueryResult.putAll(selectResult); + } else { + auxiliaryQuery = genSelectExpression(null, null, null); + auxiliaryQueryString = CockroachDBVisitor.asString(auxiliaryQuery); + auxiliaryQueryResult = getQueryResult(auxiliaryQueryString, state); + } + } else { + auxiliaryQuery = genSimpleSelect(); + auxiliaryQueryString = CockroachDBVisitor.asString(auxiliaryQuery); + auxiliaryQueryResult.putAll(selectResult); + } + + + CockroachDBSelect originalQuery = null; + + Map> foldedResult = new HashMap<>(); + Map> originalResult = new HashMap<>(); + + // dependent expression + if (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) { + CockroachDBExpressionBag specificCondition = new CockroachDBExpressionBag(this.foldedExpr); + originalQuery = this.genSelectExpression(null, specificCondition, foldedExpressionReturnType); + originalQueryString = CockroachDBVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + specificCondition.updateInnerExpr(this.constantResOfFoldedExpr); + foldedQueryString = CockroachDBVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + // independent expression + // empty result, put the inner query in (NOT) EXIST + else if (auxiliaryQueryResult.size() == 0 || auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().iterator().next()).size() == 0) { + boolean isNegated = Randomly.getBoolean() ? false : true; + + // original query + CockroachDBExists existExpr = new CockroachDBExists(auxiliaryQuery, isNegated); + CockroachDBExpressionBag specificCondition = new CockroachDBExpressionBag(existExpr); + + originalQuery = this.genSelectExpression(null, specificCondition, foldedExpressionReturnType); + originalQueryString = CockroachDBVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + CockroachDBExpression equivalentExpr = CockroachDBConstant.createBooleanConstant(isNegated); + specificCondition.updateInnerExpr(equivalentExpr); + foldedQueryString = CockroachDBVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + // Scalar Subquery: 1 column and 1 row, consider the inner query as a constant + else if (auxiliaryQueryResult.size() == 1 && auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]).size() == 1 && Randomly.getBoolean()) { + // original query + CockroachDBExpressionBag specificCondition = new CockroachDBExpressionBag(auxiliaryQuery); + + originalQuery = this.genSelectExpression(null, specificCondition, getColumnTypeFromSelect(auxiliaryQuery).get(0)); + originalQueryString = CockroachDBVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + CockroachDBExpression equivalentExpr = auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]).get(0); + specificCondition.updateInnerExpr(equivalentExpr); + foldedQueryString = CockroachDBVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + // one column + // there are bugs about `IN` and not been fixed + else if (auxiliaryQueryResult.size() == 1 && Randomly.getBoolean()) { + // original query + List columns = s.getRandomTableNonEmptyTables().getColumns(); + CockroachDBColumnReference selectedColumn = new CockroachDBColumnReference(Randomly.fromList(columns)); + CockroachDBTable selectedTable = selectedColumn.getColumn().getTable(); + CockroachDBTableReference selectedTableRef = new CockroachDBTableReference(selectedTable); + CockroachDBExpressionBag tableBag = new CockroachDBExpressionBag(selectedTableRef); + + CockroachDBInOperation optInOperation = new CockroachDBInOperation(selectedColumn, Arrays.asList(auxiliaryQuery)); + CockroachDBExpressionBag specificCondition = new CockroachDBExpressionBag(optInOperation); + + originalQuery = this.genSelectExpression(tableBag, specificCondition, CockroachDBDataType.BOOL.get()); + originalQueryString = CockroachDBVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + + // folded query + CockroachDBColumn tempColumn = new CockroachDBColumn("c0", getColumnTypeFromSelect(auxiliaryQuery).get(0), false, false); + LinkedHashMap> value = new LinkedHashMap<>(); + value.put(tempColumn, auxiliaryQueryResult.values().iterator().next()); + CockroachDBValues refValues = new CockroachDBValues(value); + CockroachDBInOperation refInOperation = new CockroachDBInOperation(selectedColumn, Arrays.asList(refValues)); + specificCondition.updateInnerExpr(refInOperation); + foldedQueryString = CockroachDBVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + // ALL + else if (auxiliaryQueryResult.size() == 1 && Randomly.getBoolean()) { + // original query + List columns = s.getRandomTableNonEmptyTables().getColumns(); + CockroachDBColumnReference selectedColumn = new CockroachDBColumnReference(Randomly.fromList(columns)); + CockroachDBTable selectedTable = selectedColumn.getColumn().getTable(); + CockroachDBTableReference selectedTableRef = new CockroachDBTableReference(selectedTable); + CockroachDBExpressionBag tableBag = new CockroachDBExpressionBag(selectedTableRef); + + CockroachDBExpression allOptLeft = genCondition(gen, null, null); + CockroachDBComparisonOperator allOperator = CockroachDBComparisonOperator.getRandom(); + while (allOperator == CockroachDBComparisonOperator.IS_DISTINCT_FROM || allOperator == CockroachDBComparisonOperator.IS_NOT_DISTINCT_FROM) { + allOperator = CockroachDBComparisonOperator.getRandom(); + } + + CockroachDBAllOperator optAllOperation = new CockroachDBAllOperator(allOptLeft, auxiliaryQuery, allOperator); + CockroachDBExpressionBag specificCondition = new CockroachDBExpressionBag(optAllOperation); + originalQuery = this.genSelectExpression(tableBag, specificCondition, CockroachDBDataType.BOOL.get()); + originalQueryString = CockroachDBVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + + // folded query + CockroachDBColumn tempColumn = new CockroachDBColumn("c0", getColumnTypeFromSelect(auxiliaryQuery).get(0), false, false); + LinkedHashMap> value = new LinkedHashMap<>(); + value.put(tempColumn, auxiliaryQueryResult.values().iterator().next()); + CockroachDBValues refValues = new CockroachDBValues(value); + CockroachDBAllOperator refAllOperation = new CockroachDBAllOperator(allOptLeft, refValues, allOperator); + specificCondition.updateInnerExpr(refAllOperation); + foldedQueryString = CockroachDBVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + + // ANY + else if (auxiliaryQueryResult.size() == 1 && Randomly.getBoolean()) { + // original query + List columns = s.getRandomTableNonEmptyTables().getColumns(); + CockroachDBColumnReference selectedColumn = new CockroachDBColumnReference(Randomly.fromList(columns)); + CockroachDBTable selectedTable = selectedColumn.getColumn().getTable(); + CockroachDBTableReference selectedTableRef = new CockroachDBTableReference(selectedTable); + CockroachDBExpressionBag tableBag = new CockroachDBExpressionBag(selectedTableRef); + + CockroachDBExpression anyOptLeft = genCondition(gen, null, null); + CockroachDBComparisonOperator anyOperator = CockroachDBComparisonOperator.getRandom(); + while (anyOperator == CockroachDBComparisonOperator.IS_DISTINCT_FROM || anyOperator == CockroachDBComparisonOperator.IS_NOT_DISTINCT_FROM) { + anyOperator = CockroachDBComparisonOperator.getRandom(); + } + + CockroachDBAnyOperator optAnyOperation = new CockroachDBAnyOperator(anyOptLeft, auxiliaryQuery, anyOperator); + CockroachDBExpressionBag specificCondition = new CockroachDBExpressionBag(optAnyOperation); + originalQuery = this.genSelectExpression(tableBag, specificCondition, CockroachDBDataType.BOOL.get()); + originalQueryString = CockroachDBVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + + // folded query + CockroachDBColumn tempColumn = new CockroachDBColumn("c0", getColumnTypeFromSelect(auxiliaryQuery).get(0), false, false); + LinkedHashMap> value = new LinkedHashMap<>(); + value.put(tempColumn, auxiliaryQueryResult.values().iterator().next()); + CockroachDBValues refValues = new CockroachDBValues(value); + CockroachDBAnyOperator refAnyOperation = new CockroachDBAnyOperator(anyOptLeft, refValues, anyOperator); + specificCondition.updateInnerExpr(refAnyOperation); + foldedQueryString = CockroachDBVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + // Row Subquery + else { + // original query + CockroachDBTable temporaryTable = this.genTemporaryTable(auxiliaryQuery, this.tempTableName); + CockroachDBTableReference tempTableRef = new CockroachDBTableReference(temporaryTable); + + LinkedHashMap> tempValue = new LinkedHashMap<>(); + for (CockroachDBColumn c : temporaryTable.getColumns()) { + tempValue.put(c, auxiliaryQueryResult.get(c.getName())); + } + CockroachDBValues values = new CockroachDBValues(tempValue); + + CockroachDBExpressionBag tempTableRefBag = new CockroachDBExpressionBag(tempTableRef); + CockroachDBTableAndColumnReference tableAndColumnRef = new CockroachDBTableAndColumnReference(temporaryTable); + CockroachDBWithClasure withClasure = new CockroachDBWithClasure(tableAndColumnRef, auxiliaryQuery); + originalQuery = genSelectExpression(tempTableRefBag, null, null); + originalQuery.setWithClause(withClasure); + originalQueryString = CockroachDBVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + if (Randomly.getBoolean()) { + // folded query: WITH table AS VALUES () + withClasure.updateRight(values); + foldedQueryString = CockroachDBVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } else if (Randomly.getBoolean()) { + // folded query: SELECT FROM () AS table + originalQuery.setWithClause(null); + CockroachDBAlias alias = null; + if (Randomly.getBoolean()) { + alias = new CockroachDBAlias(auxiliaryQuery, CockroachDBVisitor.asString(tempTableRef)); + } else { + alias = new CockroachDBAlias(values, CockroachDBVisitor.asString(tableAndColumnRef)); + } + tempTableRefBag.updateInnerExpr(alias); + foldedQueryString = CockroachDBVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } else { + // folded query: CREATE the table + try { + this.createTemporaryTable(auxiliaryQuery, this.tempTableName, CockroachDBVisitor.asString(values)); + originalQuery.setWithClause(null); + foldedQueryString = CockroachDBVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } finally { + dropTemporaryTable(this.tempTableName); + } + } + } + if (foldedResult == null || originalResult == null) { + throw new IgnoreMeException(); + } + if (foldedQueryString.equals(originalQueryString)) { + throw new IgnoreMeException(); + } + if (!compareResult(foldedResult, originalResult)) { + reproducer = null; // TODO + state.getState().getLocalState().log(auxiliaryQueryString + ";\n" + foldedQueryString + ";\n" + originalQueryString + ";"); + throw new AssertionError(auxiliaryQueryResult.toString() + " " + foldedResult.toString() + " " + originalResult.toString()); + } + + } + + private CockroachDBSelect genSelectExpression(CockroachDBExpressionBag tempTableRefBag, CockroachDBExpression specificCondition, CockroachDBCompositeDataType conditionType) { + CockroachDBTables randomTables = s.getRandomTableNonEmptyTables(); + logger.writeCurrent("1"); + if (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) { + logger.writeCurrent("2"); + for (CockroachDBTable t : this.tablesFromOuterContext) { + randomTables.addTable(t); + } + if (this.joinsInExpr != null) { + logger.writeCurrent("3"); + for (CockroachDBJoin j : this.joinsInExpr) { + logger.writeCurrent("4"); + CockroachDBTableReference lt = (CockroachDBTableReference) j.getLeftTable(); + if (lt == j.getLeftTable()) + logger.writeCurrent("lt is same with j.getLeftTable()"); + else + logger.writeCurrent("lt is not same with j.getLeftTable()"); + randomTables.addTable(lt.getTable()); + CockroachDBTableReference rt = (CockroachDBTableReference) j.getRightTable(); + randomTables.addTable(rt.getTable()); + } + } + } + + List columns = randomTables.getColumns(); + + CockroachDBTable tempTable = null; + CockroachDBTableReference tempTableRef = null; + + if (tempTableRefBag != null) { + tempTableRef = (CockroachDBTableReference) tempTableRefBag.getInnerExpr(); + tempTable = tempTableRef.getTable(); + } + + if (tempTable != null) { + columns.addAll(tempTable.getColumns()); + } + + gen = new CockroachDBExpressionGenerator(state).setColumns(columns); + List tables = randomTables.getTables(); + List tableRefs = tables.stream().map(t -> new CockroachDBTableReference(t)).collect(Collectors.toList()); + CockroachDBSelect select = new CockroachDBSelect(); + + if (tempTableRefBag != null) { + Boolean isContained = false; + for (CockroachDBExpression tre : tableRefs) { + CockroachDBTableReference tr = (CockroachDBTableReference) tre; + if (tr.getTable().getName().equals(tempTable.getName())) { + isContained = true; + break; + } + } + if (!isContained) { + tableRefs.add(tempTableRefBag); + } + } + + if (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) { + if (this.joinsInExpr != null) { + Iterator iterator = tableRefs.iterator(); + while (iterator.hasNext()) { + CockroachDBExpression e = iterator.next(); + if (e instanceof CockroachDBTableReference) { + CockroachDBTableReference tableRef = (CockroachDBTableReference) e; + + for (CockroachDBJoin j : this.joinsInExpr) { + CockroachDBTableReference leftTable = (CockroachDBTableReference) j.getLeftTable(); + CockroachDBTableReference rightTable = (CockroachDBTableReference) j.getRightTable(); + + if (tableRef.getTable().equals(leftTable.getTable()) || + tableRef.getTable().equals(rightTable.getTable())) { + iterator.remove(); + break; + } + } + } + } + } + } + + List joinExpressions = new ArrayList<>(); + if ((!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr))) { + if (this.joinsInExpr != null) { + joinExpressions.addAll(this.joinsInExpr); + this.joinsInExpr = null; + } + } else if (Randomly.getBoolean()) { + joinExpressions = getJoins(tableRefs, state, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null, conditionType); + } + + select.setFromList(tableRefs); + + if (joinExpressions.size() > 0) { + select.setJoinList(joinExpressions.stream().map(t -> (CockroachDBExpression) t).collect(Collectors.toList())); + } + + select.setWhereClause(genCondition(gen, specificCondition, conditionType)); + + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(genOrderBys(specificCondition, conditionType)); + } + + if (Randomly.getBoolean()) { + List selectedColumns = Randomly.nonEmptySubset(columns); + List selectedAlias = new LinkedList<>(); + for (int i = 0; i < selectedColumns.size(); ++i) { + CockroachDBColumnReference originalName = new CockroachDBColumnReference(selectedColumns.get(i)); + CockroachDBAlias columnAlias = new CockroachDBAlias(originalName, "c" + String.valueOf(i)); + selectedAlias.add(columnAlias); + } + select.setFetchColumns(selectedAlias); + } else { + CockroachDBColumn selectedColumn = Randomly.fromList(columns); + CockroachDBColumnReference aggr = new CockroachDBColumnReference(selectedColumn); + List windowFunctionList = CockroachDBAggregateFunction.getAggregates(selectedColumn.getType().getPrimitiveDataType()); + // The results of these window funciton will have different precision + if (windowFunctionList.contains(CockroachDBAggregateFunction.SQRDIFF)) { + windowFunctionList.remove(CockroachDBAggregateFunction.SQRDIFF); + } + if (windowFunctionList.contains(CockroachDBAggregateFunction.VARIANCE)) { + windowFunctionList.remove(CockroachDBAggregateFunction.VARIANCE); + } + // The results maybe have different order, which caused by query plan + if (windowFunctionList.contains(CockroachDBAggregateFunction.CONCAT_AGG)) { + windowFunctionList.remove(CockroachDBAggregateFunction.CONCAT_AGG); + } + // The tables from VALUES have default sorting method, but the table from alias may have + // specific sorting method. + if (selectedColumn.getType().getPrimitiveDataType().equals(CockroachDBDataType.STRING)) { + if (windowFunctionList.contains(CockroachDBAggregateFunction.MAX)) { + windowFunctionList.remove(CockroachDBAggregateFunction.MAX); + } + if (windowFunctionList.contains(CockroachDBAggregateFunction.MIN)) { + windowFunctionList.remove(CockroachDBAggregateFunction.MIN); + } + } + CockroachDBAggregateFunction windowFunction = Randomly.fromList(windowFunctionList); + CockroachDBExpression originalName = new CockroachDBAggregate(windowFunction, Arrays.asList(aggr)); + CockroachDBAlias columnAlias = new CockroachDBAlias(originalName, "c0"); + select.setFetchColumns(Arrays.asList(columnAlias)); + if (Randomly.getBoolean()) { + select.setGroupByExpressions(genGroupBys(columns, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null, conditionType)); + } + // there is an error about HAVING when use subquery in HAVING + // ERROR: subquery uses ungrouped column "rowid" from outer query + if (Randomly.getBoolean()) { + CockroachDBExpressionGenerator havingGen = new CockroachDBExpressionGenerator(state).setColumns(columns); + select.setHavingClause(genCondition(havingGen, (Randomly.getBooleanWithRatherLowProbability() && (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && !useCorrelatedSubqueryAsFoldedExpr))) ? specificCondition : null, conditionType)); + } + } + return select; + } + + public CockroachDBExpression genCondition(CockroachDBExpressionGenerator generator, CockroachDBExpression specificCondition, CockroachDBCompositeDataType conditionType) { + if (specificCondition == null) { + conditionType = null; + } + CockroachDBExpression randomWhereCondition = generator.generateExpression(conditionType == null ? CockroachDBDataType.BOOL.get() : conditionType); + CockroachDBExpression whereCondition = null; + if (specificCondition != null) { + if (conditionType == null) { + whereCondition = new CockroachDBBinaryLogicalOperation(randomWhereCondition, specificCondition, CockroachDBBinaryLogicalOperator.getRandom()); + } else { + switch(conditionType.getPrimitiveDataType()) { + case BOOL: + whereCondition = new CockroachDBBinaryLogicalOperation(randomWhereCondition, specificCondition, CockroachDBBinaryLogicalOperator.getRandom()); + break; + + case ARRAY: + case BIT: + case BYTES: + case DECIMAL: + case FLOAT: + case INT: + case INTERVAL: + case JSONB: + case SERIAL: + case STRING: + case TIME: + case TIMESTAMP: + case TIMESTAMPTZ: + case TIMETZ: + case VARBIT: + whereCondition = new CockroachDBBinaryComparisonOperator(randomWhereCondition, specificCondition, CockroachDBComparisonOperator.getRandom()); + break; + default: + throw new AssertionError(conditionType.toString()); + // whereCondition = new CockroachDBBinaryLogicalOperation(randomWhereCondition, specificCondition, CockroachDBBinaryLogicalOperator.getRandom()); + // break; + } + } + } else { + whereCondition = randomWhereCondition; + } + return whereCondition; + } + + public List getJoins(List tableList, + CockroachDBGlobalState globalState, CockroachDBExpression specificCondition, CockroachDBCompositeDataType conditionType) throws AssertionError { + List joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBoolean()) { + // CockroachDBTableReference leftTable = (CockroachDBTableReference) tableList.remove(0); + CockroachDBExpression leftExpr = tableList.remove(0); + CockroachDBTableReference leftTable = null; + if (leftExpr instanceof CockroachDBTableReference) { + leftTable = (CockroachDBTableReference) leftExpr; + } else if (leftExpr instanceof CockroachDBExpressionBag) { + leftTable = (CockroachDBTableReference) ((CockroachDBExpressionBag) leftExpr).getInnerExpr(); + } else { + throw new AssertionError(); + } + + // CockroachDBTableReference rightTable = (CockroachDBTableReference) tableList.remove(0); + CockroachDBExpression rightExpr = tableList.remove(0); + CockroachDBTableReference rightTable = null; + if (rightExpr instanceof CockroachDBTableReference) { + rightTable = (CockroachDBTableReference) rightExpr; + } else if (rightExpr instanceof CockroachDBExpressionBag) { + rightTable = (CockroachDBTableReference) ((CockroachDBExpressionBag) rightExpr).getInnerExpr(); + } else { + throw new AssertionError(); + } + + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + CockroachDBExpressionGenerator joinGen = new CockroachDBExpressionGenerator(globalState) + .setColumns(columns); + joinExpressions.add(CockroachDBJoin.createJoin(leftExpr, rightExpr, CockroachDBJoin.JoinType.getRandom(), joinGen.generateExpression(CockroachDBDataType.BOOL.get()))); + } + return joinExpressions; + } + + public List genOrderBys(CockroachDBExpression specificCondition, CockroachDBCompositeDataType conditionType) { + List orderingTerms = new ArrayList<>(); + int nr = 1; + while (Randomly.getBooleanWithSmallProbability()) { + nr++; + } + for (int i = 0; i < nr; i++) { + CockroachDBExpression expr = genCondition(gen, specificCondition, conditionType); + if (Randomly.getBoolean()) { + expr = new CockroachDBOrderingTerm(expr, Randomly.getBoolean()); + } + orderingTerms.add(expr); + } + return orderingTerms; + } + + private List genGroupBys(List columns, CockroachDBExpression specificCondition, CockroachDBCompositeDataType conditionType) { + CockroachDBExpressionGenerator groupByGen = new CockroachDBExpressionGenerator(state).setColumns(columns); + int exprNum = Randomly.smallNumber() + 1; + List newExpressions = new ArrayList<>(); + for (int i = 0; i < exprNum; ++i) { + // TODO: here we need group by condition that return integer but not boolean + CockroachDBExpression condition = genCondition(groupByGen, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null, conditionType); + newExpressions.add(condition); + } + return newExpressions; + } + + private Map> getQueryResult(String queryString, CockroachDBGlobalState state) throws SQLException { + Map> result = new LinkedHashMap<>(); + if (options.logEachSelect()) { + logger.writeCurrent(queryString); + } + Statement stmt = null; + try { + stmt = this.con.createStatement(); + ResultSet rs = null; + try { + rs = stmt.executeQuery(queryString); + ResultSetMetaData metaData = rs.getMetaData(); + Integer columnCount = metaData.getColumnCount(); + Map idxNameMap = new HashMap<>(); + for (int i = 1; i <= columnCount; i++) { + result.put("c" + String.valueOf(i-1), new ArrayList<>()); + idxNameMap.put(i, "c" + String.valueOf(i-1)); + } + + int resultRows = 0; + while (rs.next()) { + for (int i = 1; i <= columnCount; i++) { + try { + Object value = rs.getObject(i); + CockroachDBConstant constant; + if (rs.wasNull()) { + constant = CockroachDBConstant.createNullConstant(); + } + else if (value instanceof Boolean) { + constant = CockroachDBConstant.createBooleanConstant((Boolean) value); + } + else if (value instanceof Integer) { + constant = CockroachDBConstant.createIntConstant(Long.valueOf((Integer) value)); + } else if (value instanceof Short) { + constant = CockroachDBConstant.createIntConstant(Long.valueOf((Short) value)); + } else if (value instanceof Long) { + constant = CockroachDBConstant.createIntConstant((Long) value); + } else if (value instanceof Double) { + constant = CockroachDBConstant.createFloatConstant((Double) value); + } + // Usually get wrong value of bit + // else if (value instanceof PGobject) { + // constant = (CockroachDBConstant) CockroachDBConstant.createBitConstant(Long.valueOf(((PGobject) value).getValue())); + // } + + // Current there is a bug about Time and Timestamp, we just skip it now. + // else if (value instanceof Timestamp) { + // constant = (CockroachDBConstant) CockroachDBConstant.createTimestampConstant(((Timestamp) value).getTime()); + // } else if (value instanceof Time) { + // constant = (CockroachDBConstant) CockroachDBConstant.createTimeConstant(((Time) value).getTime()); + // } + else if (value instanceof BigDecimal) { + constant = CockroachDBConstant.createDecimalConstant((BigDecimal) value); + } // BigDecimal should at last + + // else if (value instanceof PgArray) { + // throw new IgnoreMeException(); + // } + else if (value instanceof String) { + constant = CockroachDBConstant.createStringConstant((String) value); + } + else if (value == null) { + constant = CockroachDBConstant.createNullConstant(); + } else { + throw new IgnoreMeException(); + } + List v = result.get(idxNameMap.get(i)); + v.add(constant); + } catch (SQLException e) { + System.out.println(e.getMessage()); + throw new IgnoreMeException(); + } + } + ++resultRows; + if (resultRows > 100) { + throw new IgnoreMeException(); + } + } + rs.close(); + Main.nrSuccessfulActions.addAndGet(1); + } catch (SQLException e) { + Main.nrUnsuccessfulActions.addAndGet(1); + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + state.getState().getLocalState().log(queryString); + throw new AssertionError(e.getMessage()); + } + } finally { + if (rs != null) { + rs.close(); + } + } + } finally { + if (stmt != null) { + stmt.close(); + } + } + + return result; + } + + private CockroachDBTable genTemporaryTable(CockroachDBSelect select, String tableName) { + List fetchColumns = select.getFetchColumns(); + int columnNumber = fetchColumns.size(); + Map idxTypeMap = getColumnTypeFromSelect(select); + + List databaseColumns = new ArrayList<>(); + for (int i = 0; i < columnNumber; ++i) { + String columnName = "c" + String.valueOf(i); + CockroachDBColumn column = new CockroachDBColumn(columnName, idxTypeMap.get(i), false, false); + databaseColumns.add(column); + } + CockroachDBTable table = new CockroachDBTable(tableName, databaseColumns, null, false); + for (CockroachDBColumn c : databaseColumns) { + c.setTable(table); + } + + return table; + } + + private CockroachDBTable createTemporaryTable(CockroachDBSelect select, String tableName, String valueString) throws SQLException { + List fetchColumns = select.getFetchColumns(); + int columnNumber = fetchColumns.size(); + Map idxTypeMap = getColumnTypeFromSelect(select); + + StringBuilder sb = new StringBuilder(); + sb.append("CREATE TABLE " + tableName + " ("); + for (int i = 0; i < columnNumber; ++i) { + String columnTypeName = ""; + if (idxTypeMap.get(i) != null) { + columnTypeName = idxTypeMap.get(i).toString(); + } + sb.append("c" + String.valueOf(i) + " " + columnTypeName + ", "); + // sb.append("c" + String.valueOf(i) + " " + idxTypeMap.get(i+1) + ", "); + // sb.append("c" + String.valueOf(i) + ", "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append(");"); + String crateTableString = sb.toString(); + if (options.logEachSelect()) { + logger.writeCurrent(crateTableString); + } + Statement stmt = null; + try { + stmt = this.con.createStatement(); + try { + stmt.execute(crateTableString); + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + state.getState().getLocalState().log(crateTableString); + throw new AssertionError(e.getMessage()); + } + } + } finally { + if (stmt != null) { + stmt.close(); + } + } + + String selectString = CockroachDBVisitor.asString(select); + StringBuilder sb2 = new StringBuilder(); + if (Randomly.getBoolean()) { + sb2.append("INSERT INTO " + tableName + " "+ selectString); + } else { + sb2.append("INSERT INTO " + tableName + " "+ valueString); + } + + String insertValueString = sb2.toString(); + if (options.logEachSelect()) { + logger.writeCurrent(insertValueString); + } + stmt = null; + try { + stmt = this.con.createStatement(); + try { + stmt.execute(insertValueString); + } catch (SQLException e) { + dropTemporaryTable(tableName); + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + state.getState().getLocalState().log(insertValueString); + throw new AssertionError(e.getMessage()); + } + } + } finally { + if (stmt != null) { + stmt.close(); + } + } + + List databaseColumns = new ArrayList<>(); + for (int i = 0; i < columnNumber; ++i) { + String columnName = "c" + String.valueOf(i); + CockroachDBColumn column = new CockroachDBColumn(columnName, idxTypeMap.get(i), false, false); + databaseColumns.add(column); + } + CockroachDBTable table = new CockroachDBTable(tableName, databaseColumns, null, false); + for (CockroachDBColumn c : databaseColumns) { + c.setTable(table); + } + + return table; + } + + private void dropTemporaryTable(String tableName) throws SQLException { + String dropString = "DROP TABLE " + tableName + ";"; + if (options.logEachSelect()) { + logger.writeCurrent(dropString); + } + Statement stmt = null; + try { + stmt = this.con.createStatement(); + try { + stmt.execute(dropString); + } catch (SQLException e) { + throw new IgnoreMeException(); + } + } finally { + if (stmt != null) { + stmt.close(); + } + } + } + + private Map getColumnTypeFromSelect(CockroachDBSelect select) { + CockroachDBSelect newSelect = new CockroachDBSelect(select); + + List fetchColumns = newSelect.getFetchColumns(); + List newFetchColumns = new ArrayList<>(); + + CockroachDBSelect innerQuery = null; + CockroachDBExpression innerQueryFetchColumn = null; + + for(CockroachDBExpression column : fetchColumns) { + CockroachDBAlias columnAlias = (CockroachDBAlias) column; + CockroachDBExpression columnExpr = columnAlias.getExpression(); + if (columnExpr instanceof CockroachDBSelect) { + innerQuery = (CockroachDBSelect) columnExpr; + List innerQueryFetchColumns = innerQuery.getFetchColumns(); + List innerQueryNewFetchColumns = new ArrayList<>(); + + for (CockroachDBExpression innerQueryColumn : innerQueryFetchColumns) { + innerQueryFetchColumn = innerQueryColumn; + CockroachDBExpression typeofColumnForInnerQuery = new CockroachDBTypeof(innerQueryColumn); + innerQueryNewFetchColumns.add(typeofColumnForInnerQuery); + } + innerQuery.setFetchColumns(innerQueryNewFetchColumns); + CockroachDBAlias subqueryTypeAlias = new CockroachDBAlias(innerQuery, "innerType"); + newFetchColumns.add(subqueryTypeAlias); + } + else { + CockroachDBExpression typeofColumn = new CockroachDBTypeof(columnAlias.getExpression()); + newFetchColumns.add(typeofColumn); + } + } + + newSelect.setFetchColumns(newFetchColumns); + Map> typeResult = null; + try { + typeResult = getQueryResult(CockroachDBVisitor.asString(newSelect), state); + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(e.getMessage()); + } + } + + if (typeResult == null) { + throw new IgnoreMeException(); + } + Map idxTypeMap = new HashMap<>(); + for (int i = 0; i < typeResult.size(); ++i) { + String columnName = "c" + String.valueOf(i); + CockroachDBExpression t = typeResult.get(columnName).get(0); + String typeName = ""; + if (t instanceof CockroachDBTextConstant) { + CockroachDBTextConstant tString = (CockroachDBTextConstant) t; + typeName = tString.getValue(); + } else { + typeName = "unknown"; + } + + CockroachDBCompositeDataType cType = new CockroachDBCompositeDataType(typeName); + idxTypeMap.put(i, cType); + } + + // revert innerquery + if (innerQuery != null) { + innerQuery.setFetchColumns(Arrays.asList(innerQueryFetchColumn)); + } + + // the type of NULL is unknown, so we need to analysis the real type + for (Map.Entry entry : idxTypeMap.entrySet()) { + Integer index = entry.getKey(); + String typeName = entry.getValue().toString(); + if (typeName.equals("unknown")) { + CockroachDBExpression column = fetchColumns.get(index); + CockroachDBCompositeDataType columnType = null; + if (column instanceof CockroachDBColumnReference) { + CockroachDBColumnReference c = (CockroachDBColumnReference) column; + columnType = c.getColumn().getType(); + } else if (column instanceof CockroachDBAlias) { + CockroachDBAlias a = (CockroachDBAlias) column; + CockroachDBExpression left = a.getExpression(); + if (left instanceof CockroachDBColumnReference) { + CockroachDBColumnReference c = (CockroachDBColumnReference) left; + columnType = c.getColumn().getType(); + } else if (left instanceof CockroachDBAggregate) { + CockroachDBAggregate aggr = (CockroachDBAggregate) left; + List aggrExprs = aggr.getExpr(); + CockroachDBExpression aggrExpr = aggrExprs.get(0); + if (aggrExpr instanceof CockroachDBColumnReference) { + CockroachDBColumnReference c = (CockroachDBColumnReference) aggrExpr; + columnType = c.getColumn().getType(); + } + } else if (left instanceof CockroachDBSelect) { + CockroachDBSelect sub = (CockroachDBSelect) left; + CockroachDBExpression subFetchColumn = sub.getFetchColumns().get(0); + if (subFetchColumn instanceof CockroachDBAggregate) { + CockroachDBAggregate subAggr = (CockroachDBAggregate) subFetchColumn; + CockroachDBExpression cr = subAggr.getExpr().get(0); + if (cr instanceof CockroachDBColumnReference) { + columnType = ((CockroachDBColumnReference) cr).getColumn().getType(); + } + } + } + } + if (columnType != null) { + idxTypeMap.put(index, columnType); + } + } + } + + return idxTypeMap; + } + + private boolean compareResult(Map> r1, Map> r2) { + if (r1.size() != r2.size()) { + return false; + } + for (Map.Entry < String, List > entry: r1.entrySet()) { + String currentKey = entry.getKey(); + if (!r2.containsKey(currentKey)) { + return false; + } + List v1= entry.getValue(); + List v2= r2.get(currentKey); + if (v1.size() != v2.size()) { + return false; + } + // TODO: sometimes the float value has different type, such as float and Decimal, fixed + List v1Value = new ArrayList<>(v1.stream().map(c -> c.toStringForComparison()).collect(Collectors.toList())); + List v2Value = new ArrayList<>(v2.stream().map(c -> c.toStringForComparison()).collect(Collectors.toList())); + Collections.sort(v1Value); + Collections.sort(v2Value); + if (!v1Value.equals(v2Value)) { + state.getState().getLocalState().log(v1Value.toString() + "\n" + v2Value.toString() + "\n"); + return false; + } + } + return true; + } + + private CockroachDBSelect genSimpleSelect() { + CockroachDBTables tables = s.getRandomTableNonEmptyTables(); + + tablesFromOuterContext = tables.getTables(); + List tableL = tables.getTables().stream().map(t -> new CockroachDBTableReference(t)) + .collect(Collectors.toList()); + CockroachDBExpressionGenerator exprGen = new CockroachDBExpressionGenerator(state).setColumns(tables.getColumns()); + this.foldedExpr = exprGen.generateExpression(CockroachDBDataType.BOOL.get()); + + CockroachDBSelect select = new CockroachDBSelect(); + + if (Randomly.getBoolean()) { + List joins = getJoins(tableL, state, null, null); + if (joins.size() > 0) { + select.setJoinClauses(joins); + this.joinsInExpr = joins; + } + } + + select.setFromList(tableL); + + List fetchColumns = new ArrayList<>(); + int columnIdx = 0; + for (CockroachDBColumn c : tables.getColumns()) { + CockroachDBColumnReference cRef = new CockroachDBColumnReference(c); + CockroachDBAlias cAlias = new CockroachDBAlias(cRef, "c" + String.valueOf(columnIdx)); + fetchColumns.add(cAlias); + columnIdx++; + } + + // add the expression as last fetch column + CockroachDBAlias eAlias = new CockroachDBAlias(this.foldedExpr, "c" + String.valueOf(columnIdx)); + fetchColumns.add(eAlias); + + select.setFetchColumns(fetchColumns); + + Map> queryRes = null; + try { + queryRes = getQueryResult(CockroachDBVisitor.asString(select), state); + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(e.getMessage()); + } + } + // just ignore the empty result, because of the empty table + if (queryRes.get("c0").size() == 0) { + throw new IgnoreMeException(); + } + + // save the result first + selectResult.clear(); + selectResult.putAll(queryRes); + + // get the constant corresponding to each row from results + List summary = queryRes.remove("c" + String.valueOf(columnIdx)); + + Boolean emptyRes = queryRes.get(queryRes.keySet().iterator().next()).size() == 0; + Map columnType = null; + if (!emptyRes) { + columnType = getColumnTypeFromSelect(select); + } + + LinkedHashMap> dbstate = new LinkedHashMap<>(); + // do not put the last fetch column to values + for (int i = 0; i < fetchColumns.size() - 1; ++i) { + CockroachDBAlias cAlias = (CockroachDBAlias) fetchColumns.get(i); + CockroachDBColumnReference cRef = (CockroachDBColumnReference) cAlias.getExpression(); + String columnName = cAlias.getAlias(); + dbstate.put(cRef, queryRes.get(columnName)); + } + + foldedExpressionReturnType = columnType.get(fetchColumns.size() - 1); + + this.constantResOfFoldedExpr = new CockroachDBResultMap(dbstate, summary, foldedExpressionReturnType); + + return select; + } + + private CockroachDBSelect genSelectWithCorrelatedSubquery() { + CockroachDBTables outerQueryRandomTables = s.getRandomTableNonEmptyTables(); + CockroachDBTables innerQueryRandomTables = s.getRandomTableNonEmptyTables(); + + List innerQueryFromTables = new ArrayList<>(); + for (CockroachDBTable t : innerQueryRandomTables.getTables()) { + if (!outerQueryRandomTables.isContained(t)) { + innerQueryFromTables.add(new CockroachDBTableReference(t)); + } + } + for (CockroachDBTable t : outerQueryRandomTables.getTables()) { + if (innerQueryRandomTables.isContained(t)) { + innerQueryRandomTables.removeTable(t); + + List newColumns = new ArrayList<>(); + for (CockroachDBColumn c : t.getColumns()) { + CockroachDBColumn newColumn = new CockroachDBColumn(c.getName(), c.getType(), false, false); + newColumns.add(newColumn); + } + CockroachDBTable newTable = new CockroachDBTable(t.getName() + "a", newColumns, null, false); + for (CockroachDBColumn c : newColumns) { + c.setTable(newTable); + } + innerQueryRandomTables.addTable(newTable); + + CockroachDBAlias alias = new CockroachDBAlias(new CockroachDBTableReference(t), newTable.getName()); + innerQueryFromTables.add(alias); + } + } + + List innerQueryColumns = new ArrayList<>(); + innerQueryColumns.addAll(innerQueryRandomTables.getColumns()); + innerQueryColumns.addAll(outerQueryRandomTables.getColumns()); + gen = new CockroachDBExpressionGenerator(state).setColumns(innerQueryColumns); + + CockroachDBSelect innerQuery = new CockroachDBSelect(); + innerQuery.setFromList(innerQueryFromTables); + + CockroachDBExpression innerQueryWhereCondition = gen.generateExpression(CockroachDBDataType.BOOL.get()); + innerQuery.setWhereClause(innerQueryWhereCondition); + + // use aggregate function in fetch column + CockroachDBColumnReference innerQueryAggr = new CockroachDBColumnReference(Randomly.fromList(innerQueryRandomTables.getColumns())); + List windowFunctionList = CockroachDBAggregateFunction.getAggregates(innerQueryAggr.getColumn().getType().getPrimitiveDataType()); + // The results of these window funciton will have differentprecision + if (windowFunctionList.contains(CockroachDBAggregateFunction.SQRDIFF)) { + windowFunctionList.remove(CockroachDBAggregateFunction.SQRDIFF); + } + if (windowFunctionList.contains(CockroachDBAggregateFunction.VARIANCE)) { + windowFunctionList.remove(CockroachDBAggregateFunction.VARIANCE); + } + // The results maybe have different order, which caused byquery plan + if (windowFunctionList.contains(CockroachDBAggregateFunction.CONCAT_AGG)) { + windowFunctionList.remove(CockroachDBAggregateFunction.CONCAT_AGG); + } + // The tables from VALUES have default sorting method, butthe table from alias may have + // specific sorting method. + if (innerQueryAggr.getColumn().getType().getPrimitiveDataType().equals(CockroachDBDataType.STRING)) { + if (windowFunctionList.contains(CockroachDBAggregateFunction.MAX)) { + windowFunctionList.remove(CockroachDBAggregateFunction.MAX); + } + if (windowFunctionList.contains(CockroachDBAggregateFunction.MIN)) { + windowFunctionList.remove(CockroachDBAggregateFunction.MIN); + } + } + CockroachDBAggregateFunction windowFunction = Randomly.fromList(windowFunctionList); + CockroachDBExpression innerQueryAggrName = new CockroachDBAggregate(windowFunction, Arrays.asList(innerQueryAggr)); + innerQuery.setFetchColumns(Arrays.asList(innerQueryAggrName)); + + this.foldedExpr = innerQuery; + + // outer query + CockroachDBSelect outerQuery = new CockroachDBSelect(); + List outerQueryFromTableRefs = outerQueryRandomTables.getTables().stream().map(t -> new CockroachDBTableReference(t)).collect(Collectors.toList()); + outerQuery.setFromList(outerQueryFromTableRefs); + tablesFromOuterContext = outerQueryRandomTables.getTables(); + + List fetchColumns = new ArrayList<>(); + int columnIdx = 0; + for (CockroachDBColumn c : outerQueryRandomTables.getColumns()) { + CockroachDBColumnReference cRef = new CockroachDBColumnReference(c); + CockroachDBAlias cAlias = new CockroachDBAlias(cRef, "c" + String.valueOf(columnIdx)); + fetchColumns.add(cAlias); + columnIdx++; + } + + // add the expression as last fetch column + CockroachDBAlias subqueryAlias = new CockroachDBAlias(innerQuery, "c" + String.valueOf(columnIdx)); + fetchColumns.add(subqueryAlias); + + outerQuery.setFetchColumns(fetchColumns); + + originalQueryString = CockroachDBVisitor.asString(outerQuery); + + + Map> queryRes = null; + try { + queryRes = getQueryResult(originalQueryString, state); + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(e.getMessage()); + } + } + // just ignore the empty result, because of the empty table + if (queryRes.get("c0").size() == 0) { + throw new IgnoreMeException(); + } + + // save the result first + selectResult.clear(); + selectResult.putAll(queryRes); + + // get the constant corresponding to each row from results + List summary = queryRes.remove("c" + String.valueOf(columnIdx)); + + Boolean emptyRes = queryRes.get(queryRes.keySet().iterator().next()).size() == 0; + Map columnType = null; + if (!emptyRes) { + columnType = getColumnTypeFromSelect(outerQuery); + } + + LinkedHashMap> dbstate = new LinkedHashMap<>(); + // do not put the last fetch column to values + for (int i = 0; i < fetchColumns.size() - 1; ++i) { + CockroachDBAlias cAlias = (CockroachDBAlias) fetchColumns.get(i); + CockroachDBColumnReference cRef = (CockroachDBColumnReference) cAlias.getExpression(); + String columnName = cAlias.getAlias(); + dbstate.put(cRef, queryRes.get(columnName)); + } + + foldedExpressionReturnType = columnType.get(fetchColumns.size() - 1); + + this.constantResOfFoldedExpr = new CockroachDBResultMap(dbstate, summary, foldedExpressionReturnType); + + return outerQuery; + } + + Boolean isEmptyTable(CockroachDBTable t) throws SQLException { + String queryString = "SELECT * FROM " + CockroachDBVisitor.asString(new CockroachDBTableReference(t)) + ";"; + int resultRows = 0; + Statement stmt = null; + try { + stmt = this.con.createStatement(); + ResultSet rs = null; + try { + rs = stmt.executeQuery(queryString); + while (rs.next()) { + ++resultRows; + } + rs.close(); + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + state.getState().getLocalState().log(queryString); + throw new AssertionError(e.getMessage()); + } + } finally { + if (rs != null) { + rs.close(); + } + } + } finally { + if (stmt != null) { + stmt.close(); + } + } + return resultRows == 0; + } + + public boolean useSubquery() { + if (this.state.getDbmsSpecificOptions().coddTestModel.equals("random")) { + return Randomly.getBoolean(); + } else if (this.state.getDbmsSpecificOptions().coddTestModel.equals("expression")) { + return false; + } else if (this.state.getDbmsSpecificOptions().coddTestModel.equals("subquery")) { + return true; + } else { + System.out.printf("Wrong option of --coddtest-model, should be one of: random, expression, subquery"); + System.exit(1); + return false; + } + } + + public boolean useCorrelatedSubquery() { + return Randomly.getBoolean(); + } + + @Override + public String getLastQueryString() { + return originalQueryString; + } + + @Override + public Reproducer getLastReproducer() { + return reproducer; + } +} From 0a8f9a13203ae44073b283f7ab65172858653541 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Mon, 13 Jan 2025 16:21:19 +0800 Subject: [PATCH 06/16] fix some false positive in CODDTest for DuckDB --- src/sqlancer/duckdb/DuckDBErrors.java | 1 + .../duckdb/DuckDBToStringVisitor.java | 8 + .../duckdb/test/DuckDBCODDTestOracle.java | 147 +++++++++++++++--- 3 files changed, 132 insertions(+), 24 deletions(-) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index ec3f7a337..d8ca8789b 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -69,6 +69,7 @@ public static List getExpressionErrors() { errors.add("GROUP BY term out of range - should be between"); errors.add("INTERNAL Error: Failed to bind column reference"); errors.add("Binder Error: Aggregate with only constant parameters has to be bound in the root subquery"); + errors.add("COLLATE can only be applied to varchar columns"); return errors; diff --git a/src/sqlancer/duckdb/DuckDBToStringVisitor.java b/src/sqlancer/duckdb/DuckDBToStringVisitor.java index e43cd0d3b..c8e21686c 100644 --- a/src/sqlancer/duckdb/DuckDBToStringVisitor.java +++ b/src/sqlancer/duckdb/DuckDBToStringVisitor.java @@ -127,6 +127,14 @@ public void visit(DuckDBResultMap expr) { visit(columnRef); if (dbstate.get(columnRef).get(i) instanceof DuckDBNullConstant) { sb.append(" IS NULL"); + } else if (dbstate.get(columnRef).get(i) instanceof DuckDBConstantWithType) { + DuckDBConstantWithType ct = (DuckDBConstantWithType) dbstate.get(columnRef).get(i); + if (ct.getConstant() instanceof DuckDBNullConstant) { + sb.append(" IS NULL"); + } else { + sb.append(" = "); + visit(dbstate.get(columnRef).get(i)); + } } else { sb.append(" = "); visit(dbstate.get(columnRef).get(i)); diff --git a/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java b/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java index b0d3d5850..62389b2d7 100644 --- a/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java +++ b/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java @@ -91,6 +91,9 @@ public DuckDBCODDTestOracle(DuckDBGlobalState globalState) { public void check() throws Exception { reproducer = null; + joinsInExpr = null; + tablesFromOuterContext.clear(); + useSubqueryAsFoldedExpr = useSubquery(); useCorrelatedSubqueryAsFoldedExpr = useCorrelatedSubquery(); @@ -320,9 +323,11 @@ private DuckDBSelect genSelectExpression(DuckDBExpressionBag tableBag, DuckDBExp DuckDBSelect select = new DuckDBSelect(); List joins = new ArrayList<>(); - if ((!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) && this.joinsInExpr != null) { - joins.addAll(this.joinsInExpr); - this.joinsInExpr = null; + if ((!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr))) { + if (this.joinsInExpr != null) { + joins.addAll(this.joinsInExpr); + this.joinsInExpr = null; + } } else if (Randomly.getBoolean()) { joins = genJoinExpression(tableRefList, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null); @@ -475,6 +480,18 @@ private DuckDBSelect genSelectWithCorrelatedSubquery() { // get the constant corresponding to each row from results List summary = queryRes.remove("c" + String.valueOf(columnIdx)); + Map columnsType = getColumnTypeFromSelect(outerQuery); + + DuckDBCompositeDataType exprType = columnsType.get(outerQueryFetchColumns.size() -1 ); + if (exprType.toString().equals("REAL") || exprType.toString().startsWith("FLOAT")) { + throw new IgnoreMeException(); + } + + List constantRes = new ArrayList<>(); + for (DuckDBExpression e : summary) { + DuckDBConstant c = (DuckDBConstant) e; + constantRes.add(new DuckDBConstantWithType(c, exprType)); + } LinkedHashMap> dbstate = new LinkedHashMap<>(); @@ -483,9 +500,17 @@ private DuckDBSelect genSelectWithCorrelatedSubquery() { DuckDBAlias cAlias = (DuckDBAlias) outerQueryFetchColumns.get(i); DuckDBColumnReference cRef = (DuckDBColumnReference) cAlias.getExpr(); String columnName = cAlias.getAlias(); - dbstate.put(cRef, queryRes.get(columnName)); + List constants = new ArrayList<>(); + for (DuckDBExpression e : queryRes.get(columnName)) { + DuckDBConstant c = (DuckDBConstant) e; + if (columnsType.get(i).toString().equals("REAL") || columnsType.get(i).toString().startsWith("FLOAT")) { + throw new IgnoreMeException(); + } + constants.add(new DuckDBConstantWithType(c, columnsType.get(i))); + } + dbstate.put(cRef, constants); } - this.constantResOfFoldedExpr = new DuckDBResultMap(dbstate, summary); + this.constantResOfFoldedExpr = new DuckDBResultMap(dbstate, constantRes); return outerQuery; } @@ -553,6 +578,9 @@ private DuckDBSelect genSimpleSelect() { List summary = queryRes.remove(exprAliasName); Map columnsType = getColumnTypeFromSelect(select); DuckDBCompositeDataType exprType = columnsType.get(fetchColumns.size() -1 ); + if (exprType.toString().equals("REAL") || exprType.toString().startsWith("FLOAT")) { + throw new IgnoreMeException(); + } List constantRes = new ArrayList<>(); for (DuckDBExpression e : summary) { DuckDBConstant c = (DuckDBConstant) e; @@ -566,7 +594,15 @@ private DuckDBSelect genSimpleSelect() { DuckDBAlias cAlias = (DuckDBAlias) fetchColumns.get(i); DuckDBColumnReference cRef = (DuckDBColumnReference) cAlias.getExpr(); String columnName = cAlias.getAlias(); - dbstate.put(cRef, queryRes.get(columnName)); + List constants = new ArrayList<>(); + for (DuckDBExpression e : queryRes.get(columnName)) { + DuckDBConstant c = (DuckDBConstant) e; + if (columnsType.get(i).toString().equals("REAL") || columnsType.get(i).toString().startsWith("FLOAT")) { + throw new IgnoreMeException(); + } + constants.add(new DuckDBConstantWithType(c, columnsType.get(i))); + } + dbstate.put(cRef, constants); } this.constantResOfFoldedExpr = new DuckDBResultMap(dbstate, constantRes); @@ -805,15 +841,37 @@ private void dropTemporaryTable(String tableName) throws SQLException { } private Map getColumnTypeFromSelect(DuckDBSelect select) { + DuckDBSelect newSelect = new DuckDBSelect(select); + + List fetchColumns = select.getFetchColumns(); List newFetchColumns = new ArrayList<>(); + + DuckDBSelect innerQuery = null; + DuckDBExpression innerQueryFetchColumn = null; + for(DuckDBExpression column : fetchColumns) { - newFetchColumns.add(column); DuckDBAlias columnAlias = (DuckDBAlias) column; - DuckDBTypeofNode typeofColumn = new DuckDBTypeofNode(columnAlias.getExpr()); - newFetchColumns.add(typeofColumn); + DuckDBExpression columnExpr = columnAlias.getExpr(); + if (columnExpr instanceof DuckDBSelect) { + innerQuery = (DuckDBSelect) columnExpr; + List innerQueryFetchColumns = innerQuery.getFetchColumns(); + List innerQueryNewFetchColumns = new ArrayList<>(); + for (DuckDBExpression innerQueryColumn : innerQueryFetchColumns) { + innerQueryFetchColumn = innerQueryColumn; + DuckDBExpression typeofColumnForInnerQuery = new DuckDBTypeofNode(innerQueryColumn); + innerQueryNewFetchColumns.add(typeofColumnForInnerQuery); + } + innerQuery.setFetchColumns(innerQueryNewFetchColumns); + DuckDBAlias subqueryTypeAlias = new DuckDBAlias(innerQuery, "innerType"); + newFetchColumns.add(subqueryTypeAlias); + } + else { + DuckDBTypeofNode typeofColumn = new DuckDBTypeofNode(columnAlias.getExpr()); + newFetchColumns.add(typeofColumn); + } } - DuckDBSelect newSelect = new DuckDBSelect(select); + newSelect.setFetchColumns(newFetchColumns); Map> typeResult = null; try { @@ -829,14 +887,65 @@ private Map getColumnTypeFromSelect(DuckDBSele throw new IgnoreMeException(); } Map idxTypeMap = new HashMap<>(); - for (int i = 0; i * 2 < typeResult.size(); ++i) { - String columnName = "c" + String.valueOf(i * 2 + 1); + for (int i = 0; i < typeResult.size(); ++i) { + String columnName = "c" + String.valueOf(i); DuckDBExpression t = typeResult.get(columnName).get(0); - DuckDBTextConstant tString = (DuckDBTextConstant) t; - String typeName = tString.getValue(); + String typeName = ""; + if (t instanceof DuckDBTextConstant) { + DuckDBTextConstant tString = (DuckDBTextConstant) t; + typeName = tString.getValue(); + } else { + typeName = "unknown"; + } DuckDBCompositeDataType cType = new DuckDBCompositeDataType(typeName); idxTypeMap.put(i, cType); } + + // revert innerquery + if (innerQuery != null) { + innerQuery.setFetchColumns(Arrays.asList(innerQueryFetchColumn)); + } + + for (Map.Entry entry: idxTypeMap.entrySet()) { + Integer index = entry.getKey(); + String typeName = entry.getValue().toString(); + if (typeName.equals("unknown")) { + DuckDBExpression column = fetchColumns.get(index); + DuckDBCompositeDataType columnType = null; + if (column instanceof DuckDBColumnReference) { + DuckDBColumnReference c = (DuckDBColumnReference)column; + columnType = c.getColumn().getType(); + } else if (column instanceof DuckDBAlias) { + DuckDBAlias a = (DuckDBAlias) column; + DuckDBExpression left = a.getExpr(); + if (left instanceof DuckDBColumnReference) { + DuckDBColumnReference c = (DuckDBColumnReference) left; + columnType = c.getColumn().getType(); + } else if (left instanceof DuckDBFunction) { + DuckDBFunction aggr = (DuckDBFunction) left; + List aggrExprs = aggr.getArgs(); + DuckDBExpression aggrExpr = aggrExprs.get(0); + if (aggrExpr instanceof DuckDBColumnReference) { + DuckDBColumnReference c = (DuckDBColumnReference) aggrExpr; + columnType = c.getColumn().getType(); + } + } else if (left instanceof DuckDBSelect) { + DuckDBSelect sub = (DuckDBSelect) left; + DuckDBExpression subFetchColumn = sub.getFetchColumns().get(0); + if (subFetchColumn instanceof DuckDBFunction) { + DuckDBFunction subAggr = (DuckDBFunction) subFetchColumn; + DuckDBExpression cr = subAggr.getArgs().get(0); + if (cr instanceof DuckDBColumnReference) { + columnType = ((DuckDBColumnReference) cr).getColumn().getType(); + } + } + } + } + if (columnType != null) { + idxTypeMap.put(index, columnType); + } + } + } return idxTypeMap; } @@ -896,16 +1005,6 @@ public boolean useCorrelatedSubquery() { return Randomly.getBoolean(); } - public boolean testCommonTableExpression() { - return true; - } - public boolean testDerivedTable() { - return true; - } - public boolean testInsert() { - return true; - } - @Override public String getLastQueryString() { return originalQueryString; From 9596426e6333c374e037c568aace81ae0e0370fc Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Tue, 14 Jan 2025 16:03:00 +0800 Subject: [PATCH 07/16] fix the false positive in CODDTest for SQLite --- .../sqlite3/oracle/SQLite3CODDTestOracle.java | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java index 7118f4081..c4e8d892a 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java @@ -94,6 +94,9 @@ public SQLite3CODDTestOracle(SQLite3GlobalState globalState) { public void check() throws SQLException { reproducer = null; + joinsInExpr = null; + tablesFromOuterContext.clear(); + useSubqueryAsFoldedExpr = useSubquery(); useCorrelatedSubqueryAsFoldedExpr = useCorrelatedSubquery(); @@ -168,8 +171,7 @@ else if (auxiliaryQueryResult.size() == 1 && auxiliaryQueryResult.get(auxiliaryQ foldedResult = getQueryResult(foldedQueryString, state); } // one column - else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowProbability()) { - // else if (auxiliaryQueryResult.size() == 1 && false) { + else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowProbability() && testInOperator()) { // original query List columns = s.getRandomTableNonEmptyTables().getColumns(); SQLite3ColumnName selectedColumn = new SQLite3ColumnName(Randomly.fromList(columns), null); @@ -277,9 +279,11 @@ private SQLite3Select genSelectExpression(SQLite3Table tempTable, SQLite3Express gen = new SQLite3ExpressionGenerator(state).setColumns(columns); List tables = randomTables.getTables(); List joinStatements = new ArrayList<>(); - if ((!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) && this.joinsInExpr != null) { - joinStatements.addAll(this.joinsInExpr); - this.joinsInExpr = null; + if (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) { + if (this.joinsInExpr != null) { + joinStatements.addAll(this.joinsInExpr); + this.joinsInExpr = null; + } } else if (Randomly.getBoolean()) { joinStatements = genJoinExpression(gen, tables, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null, false); @@ -950,6 +954,9 @@ public boolean testDerivedTable() { public boolean testInsert() { return false; } + public boolean testInOperator() { + return false; + } @Override public String getLastQueryString() { From d4e0f8f8050aa2fc33a699dd0705fc43f8de10e2 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Thu, 16 Jan 2025 19:28:29 +0800 Subject: [PATCH 08/16] remove a piece of redundant code --- src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java b/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java index 62389b2d7..414f6ee4e 100644 --- a/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java +++ b/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java @@ -397,11 +397,11 @@ private DuckDBSelect genSelectWithCorrelatedSubquery() { throw new IgnoreMeException(); } - for (DuckDBTable t : outerQueryRandomTables.getTables()) { - if (!innerQueryRandomTables.isContained(t)) { - tablesFromOuterContext.add(t); - } - } + // for (DuckDBTable t : outerQueryRandomTables.getTables()) { + // if (!innerQueryRandomTables.isContained(t)) { + // tablesFromOuterContext.add(t); + // } + // } List innerQueryColumns = new ArrayList<>(); innerQueryColumns.addAll(innerQueryRandomTables.getColumns()); From 412273167d8b710143954da1db0f7e89a4eec6f2 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Thu, 16 Jan 2025 19:29:14 +0800 Subject: [PATCH 09/16] remove some radundant log code in CODDTest for CockroachDB --- .../cockroachdb/oracle/CockroachDBCODDTestOracle.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBCODDTestOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBCODDTestOracle.java index fc87d79cc..5c1a5e61f 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBCODDTestOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBCODDTestOracle.java @@ -366,21 +366,13 @@ else if (auxiliaryQueryResult.size() == 1 && Randomly.getBoolean()) { private CockroachDBSelect genSelectExpression(CockroachDBExpressionBag tempTableRefBag, CockroachDBExpression specificCondition, CockroachDBCompositeDataType conditionType) { CockroachDBTables randomTables = s.getRandomTableNonEmptyTables(); - logger.writeCurrent("1"); if (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) { - logger.writeCurrent("2"); for (CockroachDBTable t : this.tablesFromOuterContext) { randomTables.addTable(t); } if (this.joinsInExpr != null) { - logger.writeCurrent("3"); for (CockroachDBJoin j : this.joinsInExpr) { - logger.writeCurrent("4"); CockroachDBTableReference lt = (CockroachDBTableReference) j.getLeftTable(); - if (lt == j.getLeftTable()) - logger.writeCurrent("lt is same with j.getLeftTable()"); - else - logger.writeCurrent("lt is not same with j.getLeftTable()"); randomTables.addTable(lt.getTable()); CockroachDBTableReference rt = (CockroachDBTableReference) j.getRightTable(); randomTables.addTable(rt.getTable()); From bc53382ba6166d5fda552d3692d6ba3da12b9283 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Thu, 16 Jan 2025 19:30:08 +0800 Subject: [PATCH 10/16] support CODDTest for TiDB --- .../tidb/TiDBExpressionGenerator.java | 9 +- src/sqlancer/tidb/TiDBOptions.java | 3 + src/sqlancer/tidb/TiDBOracleFactory.java | 8 + src/sqlancer/tidb/ast/TiDBAlias.java | 19 + src/sqlancer/tidb/ast/TiDBAllOperator.java | 26 + src/sqlancer/tidb/ast/TiDBAnyOperator.java | 26 + src/sqlancer/tidb/ast/TiDBConstant.java | 14 +- src/sqlancer/tidb/ast/TiDBExists.java | 24 + src/sqlancer/tidb/ast/TiDBExpressionBag.java | 18 + src/sqlancer/tidb/ast/TiDBInOperator.java | 20 + src/sqlancer/tidb/ast/TiDBResultMap.java | 33 + .../tidb/ast/TiDBTableAndColumnReference.java | 15 + src/sqlancer/tidb/ast/TiDBValues.java | 19 + src/sqlancer/tidb/ast/TiDBValuesRow.java | 18 + src/sqlancer/tidb/ast/TiDBWithClasure.java | 24 + .../tidb/oracle/TiDBCODDTestOracle.java | 1086 +++++++++++++++++ .../tidb/visitor/TiDBToStringVisitor.java | 235 ++++ src/sqlancer/tidb/visitor/TiDBVisitor.java | 46 + 18 files changed, 1635 insertions(+), 8 deletions(-) create mode 100644 src/sqlancer/tidb/ast/TiDBAlias.java create mode 100644 src/sqlancer/tidb/ast/TiDBAllOperator.java create mode 100644 src/sqlancer/tidb/ast/TiDBAnyOperator.java create mode 100644 src/sqlancer/tidb/ast/TiDBExists.java create mode 100644 src/sqlancer/tidb/ast/TiDBExpressionBag.java create mode 100644 src/sqlancer/tidb/ast/TiDBInOperator.java create mode 100644 src/sqlancer/tidb/ast/TiDBResultMap.java create mode 100644 src/sqlancer/tidb/ast/TiDBTableAndColumnReference.java create mode 100644 src/sqlancer/tidb/ast/TiDBValues.java create mode 100644 src/sqlancer/tidb/ast/TiDBValuesRow.java create mode 100644 src/sqlancer/tidb/ast/TiDBWithClasure.java create mode 100644 src/sqlancer/tidb/oracle/TiDBCODDTestOracle.java diff --git a/src/sqlancer/tidb/TiDBExpressionGenerator.java b/src/sqlancer/tidb/TiDBExpressionGenerator.java index ff6a529d0..5ed482b04 100644 --- a/src/sqlancer/tidb/TiDBExpressionGenerator.java +++ b/src/sqlancer/tidb/TiDBExpressionGenerator.java @@ -1,5 +1,6 @@ package sqlancer.tidb; +import java.math.BigInteger; import java.util.ArrayList; import java.util.Arrays; import java.util.List; @@ -73,7 +74,7 @@ public TiDBExpression generateConstant() { } switch (type) { case INT: - return TiDBConstant.createIntConstant(globalState.getRandomly().getInteger()); + return TiDBConstant.createIntConstant(BigInteger.valueOf(globalState.getRandomly().getInteger())); case BLOB: case TEXT: return TiDBConstant.createStringConstant(globalState.getRandomly().getString()); @@ -85,7 +86,7 @@ public TiDBExpression generateConstant() { return TiDBConstant.createStringConstant(globalState.getRandomly().getChar()); case DECIMAL: case NUMERIC: - return TiDBConstant.createIntConstant(globalState.getRandomly().getInteger()); + return TiDBConstant.createIntConstant(BigInteger.valueOf(globalState.getRandomly().getInteger())); default: throw new AssertionError(); } @@ -121,7 +122,7 @@ public TiDBExpression generateConstant(TiDBDataType type) { } switch (type) { case INT: - return TiDBConstant.createIntConstant(globalState.getRandomly().getInteger()); + return TiDBConstant.createIntConstant(BigInteger.valueOf(globalState.getRandomly().getInteger())); case BLOB: case TEXT: return TiDBConstant.createStringConstant(globalState.getRandomly().getString()); @@ -133,7 +134,7 @@ public TiDBExpression generateConstant(TiDBDataType type) { return TiDBConstant.createStringConstant(globalState.getRandomly().getChar()); case DECIMAL: case NUMERIC: - return TiDBConstant.createIntConstant(globalState.getRandomly().getInteger()); + return TiDBConstant.createIntConstant(BigInteger.valueOf(globalState.getRandomly().getInteger())); default: throw new AssertionError(); } diff --git a/src/sqlancer/tidb/TiDBOptions.java b/src/sqlancer/tidb/TiDBOptions.java index 1619832d6..c4264e765 100644 --- a/src/sqlancer/tidb/TiDBOptions.java +++ b/src/sqlancer/tidb/TiDBOptions.java @@ -29,6 +29,9 @@ public class TiDBOptions implements DBMSSpecificOptions { @Parameter(names = { "--tiflash" }, description = "Enable TiFlash") public boolean tiflash; + @Parameter(names = { "--coddtest-model" }, description = "Apply CODDTest on expression, subquery, or random") + public String coddTestModel = "random"; + @Override public List getTestOracleFactory() { return oracle; diff --git a/src/sqlancer/tidb/TiDBOracleFactory.java b/src/sqlancer/tidb/TiDBOracleFactory.java index 173ff7abd..eba74f914 100644 --- a/src/sqlancer/tidb/TiDBOracleFactory.java +++ b/src/sqlancer/tidb/TiDBOracleFactory.java @@ -12,6 +12,8 @@ import sqlancer.common.oracle.TestOracle; import sqlancer.common.query.ExpectedErrors; import sqlancer.common.query.SQLancerResultSet; +import sqlancer.tidb.TiDBProvider.TiDBGlobalState; +import sqlancer.tidb.oracle.TiDBCODDTestOracle; import sqlancer.tidb.oracle.TiDBDQPOracle; import sqlancer.tidb.oracle.TiDBTLPHavingOracle; @@ -72,6 +74,12 @@ public TestOracle create(TiDBProvider.TiDBGlobalSt throws SQLException { return new TiDBDQPOracle(globalState); } + }, + CODDTest { + @Override + public TestOracle create(TiDBGlobalState globalState) throws SQLException { + return new TiDBCODDTestOracle(globalState); + } }; } diff --git a/src/sqlancer/tidb/ast/TiDBAlias.java b/src/sqlancer/tidb/ast/TiDBAlias.java new file mode 100644 index 000000000..b1e6ba4d5 --- /dev/null +++ b/src/sqlancer/tidb/ast/TiDBAlias.java @@ -0,0 +1,19 @@ +package sqlancer.tidb.ast; + +public class TiDBAlias implements TiDBExpression { + private final TiDBExpression expr; + private final String alias; + + public TiDBAlias(TiDBExpression expr, String alias) { + this.expr = expr; + this.alias = alias; + } + + public TiDBExpression getExpression() { + return expr; + } + + public String getAlias() { + return alias; + } +} diff --git a/src/sqlancer/tidb/ast/TiDBAllOperator.java b/src/sqlancer/tidb/ast/TiDBAllOperator.java new file mode 100644 index 000000000..73e696b90 --- /dev/null +++ b/src/sqlancer/tidb/ast/TiDBAllOperator.java @@ -0,0 +1,26 @@ +package sqlancer.tidb.ast; + +import sqlancer.tidb.ast.TiDBBinaryComparisonOperation.TiDBComparisonOperator; + +public class TiDBAllOperator implements TiDBExpression { + private final TiDBExpression leftExpr; + private final TiDBExpression rightExpr; + private final TiDBComparisonOperator op; + + + public TiDBAllOperator(TiDBExpression leftExpr, TiDBExpression rightExpr, TiDBComparisonOperator op) { + this.leftExpr = leftExpr; + this.rightExpr = rightExpr; + this.op = op; + } + + public TiDBExpression getLeftExpr() { + return leftExpr; + } + public TiDBExpression getRightExpr() { + return rightExpr; + } + public String getOperator() { + return op.getTextRepresentation(); + } +} diff --git a/src/sqlancer/tidb/ast/TiDBAnyOperator.java b/src/sqlancer/tidb/ast/TiDBAnyOperator.java new file mode 100644 index 000000000..8df40ca6b --- /dev/null +++ b/src/sqlancer/tidb/ast/TiDBAnyOperator.java @@ -0,0 +1,26 @@ +package sqlancer.tidb.ast; + +import sqlancer.tidb.ast.TiDBBinaryComparisonOperation.TiDBComparisonOperator; + +public class TiDBAnyOperator implements TiDBExpression { + private final TiDBExpression leftExpr; + private final TiDBExpression rightExpr; + private final TiDBComparisonOperator op; + + + public TiDBAnyOperator(TiDBExpression leftExpr, TiDBExpression rightExpr, TiDBComparisonOperator op) { + this.leftExpr = leftExpr; + this.rightExpr = rightExpr; + this.op = op; + } + + public TiDBExpression getLeftExpr() { + return leftExpr; + } + public TiDBExpression getRightExpr() { + return rightExpr; + } + public String getOperator() { + return op.getTextRepresentation(); + } +} diff --git a/src/sqlancer/tidb/ast/TiDBConstant.java b/src/sqlancer/tidb/ast/TiDBConstant.java index 33ac634bb..db10f056e 100644 --- a/src/sqlancer/tidb/ast/TiDBConstant.java +++ b/src/sqlancer/tidb/ast/TiDBConstant.java @@ -1,5 +1,7 @@ package sqlancer.tidb.ast; +import java.math.BigInteger; + public class TiDBConstant implements TiDBExpression { private TiDBConstant() { @@ -16,9 +18,9 @@ public String toString() { public static class TiDBIntConstant extends TiDBConstant { - private final long value; + private final BigInteger value; - public TiDBIntConstant(long value) { + public TiDBIntConstant(BigInteger value) { this.value = value; } @@ -27,7 +29,7 @@ public String toString() { return String.valueOf(value); } - public long getValue() { + public BigInteger getValue() { return value; } @@ -122,7 +124,7 @@ public static TiDBDoubleConstant createFloatConstant(double val) { return new TiDBDoubleConstant(val); } - public static TiDBIntConstant createIntConstant(long val) { + public static TiDBIntConstant createIntConstant(BigInteger val) { return new TiDBIntConstant(val); } @@ -134,4 +136,8 @@ public static TiDBConstant createBooleanConstant(boolean val) { return new TiDBBooleanConstant(val); } + public static TiDBBitConstant createBitConstant(long val) { + return new TiDBBitConstant(val); + } + } diff --git a/src/sqlancer/tidb/ast/TiDBExists.java b/src/sqlancer/tidb/ast/TiDBExists.java new file mode 100644 index 000000000..7eee99194 --- /dev/null +++ b/src/sqlancer/tidb/ast/TiDBExists.java @@ -0,0 +1,24 @@ +package sqlancer.tidb.ast; + +public class TiDBExists implements TiDBExpression { + private final TiDBExpression select; + private boolean negated = false; + + public TiDBExists(TiDBExpression select, boolean negated) { + this.select = select; + this.negated = negated; + } + + public void setNegated(boolean negated) { + this.negated = negated; + } + + public boolean getNegated() { + return this.negated; + } + + public TiDBExpression getExpression() { + return select; + } + +} diff --git a/src/sqlancer/tidb/ast/TiDBExpressionBag.java b/src/sqlancer/tidb/ast/TiDBExpressionBag.java new file mode 100644 index 000000000..2668f74fe --- /dev/null +++ b/src/sqlancer/tidb/ast/TiDBExpressionBag.java @@ -0,0 +1,18 @@ +package sqlancer.tidb.ast; + +public class TiDBExpressionBag implements TiDBExpression { + private TiDBExpression innerExpr; + + public TiDBExpressionBag(TiDBExpression innerExpr) { + this.innerExpr = innerExpr; + } + + public void updateInnerExpr(TiDBExpression innerExpr) { + this.innerExpr = innerExpr; + } + + public TiDBExpression getInnerExpr() { + return innerExpr; + } + +} diff --git a/src/sqlancer/tidb/ast/TiDBInOperator.java b/src/sqlancer/tidb/ast/TiDBInOperator.java new file mode 100644 index 000000000..57557cd31 --- /dev/null +++ b/src/sqlancer/tidb/ast/TiDBInOperator.java @@ -0,0 +1,20 @@ +package sqlancer.tidb.ast; + +public class TiDBInOperator implements TiDBExpression { + private final TiDBExpression left; + private final TiDBExpression right; + + public TiDBInOperator(TiDBExpression left, TiDBExpression right) { + this.left = left; + this.right = right; + } + + public TiDBExpression getLeft() { + return left; + } + + public TiDBExpression getRight() { + return right; + } + +} diff --git a/src/sqlancer/tidb/ast/TiDBResultMap.java b/src/sqlancer/tidb/ast/TiDBResultMap.java new file mode 100644 index 000000000..4327b295e --- /dev/null +++ b/src/sqlancer/tidb/ast/TiDBResultMap.java @@ -0,0 +1,33 @@ +package sqlancer.tidb.ast; + +import java.util.LinkedHashMap; +import java.util.List; + +import sqlancer.tidb.TiDBSchema.TiDBCompositeDataType; + +public class TiDBResultMap implements TiDBExpression { + private final LinkedHashMap> DBStates; + private final List results; + TiDBCompositeDataType resultType; + + public TiDBResultMap(LinkedHashMap> s, List r, TiDBCompositeDataType rt) { + this.DBStates = s; + this.results = r; + this.resultType = rt; + if (s.get(s.keySet().iterator().next()).size() != r.size()) { + throw new AssertionError(); + } + } + + public LinkedHashMap> getDbStates() { + return this.DBStates; + } + + public List getResult() { + return this.results; + } + + public TiDBCompositeDataType getResultType() { + return this.resultType; + } +} diff --git a/src/sqlancer/tidb/ast/TiDBTableAndColumnReference.java b/src/sqlancer/tidb/ast/TiDBTableAndColumnReference.java new file mode 100644 index 000000000..d6f88b186 --- /dev/null +++ b/src/sqlancer/tidb/ast/TiDBTableAndColumnReference.java @@ -0,0 +1,15 @@ +package sqlancer.tidb.ast; + +import sqlancer.tidb.TiDBSchema.TiDBTable; + +public class TiDBTableAndColumnReference implements TiDBExpression { + private final TiDBTable table; + + public TiDBTableAndColumnReference(TiDBTable table) { + this.table = table; + } + + public TiDBTable getTable() { + return table; + } +} diff --git a/src/sqlancer/tidb/ast/TiDBValues.java b/src/sqlancer/tidb/ast/TiDBValues.java new file mode 100644 index 000000000..76f755c9c --- /dev/null +++ b/src/sqlancer/tidb/ast/TiDBValues.java @@ -0,0 +1,19 @@ +package sqlancer.tidb.ast; + +import java.util.LinkedHashMap; +import java.util.List; + +import sqlancer.tidb.TiDBSchema.TiDBColumn; + +public class TiDBValues implements TiDBExpression { + + private final LinkedHashMap> values; + + public TiDBValues(LinkedHashMap> v) { + this.values = v; + } + + public LinkedHashMap> getValues() { + return this.values; + } +} diff --git a/src/sqlancer/tidb/ast/TiDBValuesRow.java b/src/sqlancer/tidb/ast/TiDBValuesRow.java new file mode 100644 index 000000000..dd48c21d6 --- /dev/null +++ b/src/sqlancer/tidb/ast/TiDBValuesRow.java @@ -0,0 +1,18 @@ +package sqlancer.tidb.ast; + +import java.util.LinkedHashMap; +import java.util.List; + +import sqlancer.tidb.TiDBSchema.TiDBColumn; + +public class TiDBValuesRow implements TiDBExpression { + private final LinkedHashMap> values; + + public TiDBValuesRow(LinkedHashMap> values) { + this.values = values; + } + + public LinkedHashMap> getValues() { + return this.values; + } +} diff --git a/src/sqlancer/tidb/ast/TiDBWithClasure.java b/src/sqlancer/tidb/ast/TiDBWithClasure.java new file mode 100644 index 000000000..ce3d650eb --- /dev/null +++ b/src/sqlancer/tidb/ast/TiDBWithClasure.java @@ -0,0 +1,24 @@ +package sqlancer.tidb.ast; + +public class TiDBWithClasure implements TiDBExpression { + private TiDBExpression left; + private TiDBExpression right; + + public TiDBWithClasure(TiDBExpression left, TiDBExpression right) { + this.left = left; + this.right = right; + } + + public TiDBExpression getLeft() { + return this.left; + } + + public TiDBExpression getRight() { + return this.right; + } + + public void updateRight(TiDBExpression right) { + this.right = right; + } + +} diff --git a/src/sqlancer/tidb/oracle/TiDBCODDTestOracle.java b/src/sqlancer/tidb/oracle/TiDBCODDTestOracle.java new file mode 100644 index 000000000..88df9dc22 --- /dev/null +++ b/src/sqlancer/tidb/oracle/TiDBCODDTestOracle.java @@ -0,0 +1,1086 @@ +package sqlancer.tidb.oracle; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.regex.Pattern; + +import sqlancer.IgnoreMeException; +import sqlancer.Main; +import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.common.oracle.CODDTestBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.tidb.TiDBErrors; +import sqlancer.tidb.TiDBExpressionGenerator; +import sqlancer.tidb.TiDBSchema; +import sqlancer.tidb.TiDBProvider.TiDBGlobalState; +import sqlancer.tidb.TiDBSchema.TiDBColumn; +import sqlancer.tidb.TiDBSchema.TiDBCompositeDataType; +import sqlancer.tidb.TiDBSchema.TiDBDataType; +import sqlancer.tidb.TiDBSchema.TiDBTable; +import sqlancer.tidb.TiDBSchema.TiDBTables; +import sqlancer.tidb.ast.TiDBAggregate; +import sqlancer.tidb.ast.TiDBAlias; +import sqlancer.tidb.ast.TiDBAllOperator; +import sqlancer.tidb.ast.TiDBAnyOperator; +import sqlancer.tidb.ast.TiDBBinaryComparisonOperation; +import sqlancer.tidb.ast.TiDBBinaryLogicalOperation; +import sqlancer.tidb.ast.TiDBColumnReference; +import sqlancer.tidb.ast.TiDBConstant; +import sqlancer.tidb.ast.TiDBExists; +import sqlancer.tidb.ast.TiDBExpression; +import sqlancer.tidb.ast.TiDBExpressionBag; +import sqlancer.tidb.ast.TiDBInOperator; +import sqlancer.tidb.ast.TiDBJoin; +import sqlancer.tidb.ast.TiDBOrderingTerm; +import sqlancer.tidb.ast.TiDBSelect; +import sqlancer.tidb.ast.TiDBTableAndColumnReference; +import sqlancer.tidb.ast.TiDBTableReference; +import sqlancer.tidb.ast.TiDBResultMap; +import sqlancer.tidb.ast.TiDBValues; +import sqlancer.tidb.ast.TiDBValuesRow; +import sqlancer.tidb.ast.TiDBWithClasure; +import sqlancer.tidb.ast.TiDBAggregate.TiDBAggregateFunction; +import sqlancer.tidb.ast.TiDBBinaryComparisonOperation.TiDBComparisonOperator; +import sqlancer.tidb.ast.TiDBBinaryLogicalOperation.TiDBBinaryLogicalOperator; +import sqlancer.tidb.ast.TiDBJoin.NaturalJoinType; +import sqlancer.tidb.visitor.TiDBVisitor; + +public class TiDBCODDTestOracle extends CODDTestBase implements TestOracle { + + private final TiDBSchema s; + private TiDBExpressionGenerator gen; + private Reproducer reproducer; + + private String tempTableName = "temp_table"; + + private TiDBExpression foldedExpr; + private TiDBExpression constantResOfFoldedExpr; + + + private List tablesFromOuterContext = new ArrayList<>(); + private List joinsInExpr = null; + + Map> auxiliaryQueryResult = new HashMap<>(); + Map> selectResult = new HashMap<>(); + + Boolean useSubqueryAsFoldedExpr; + Boolean useCorrelatedSubqueryAsFoldedExpr; + + TiDBCompositeDataType foldedExpressionReturnType = null; + + public TiDBCODDTestOracle(TiDBGlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + TiDBErrors.addExpressionErrors(errors); + TiDBErrors.addInsertErrors(errors); + + errors.add("strconv.Atoi: parsing"); + errors.add("interface conversion: expression.Expression is *expression.ScalarFunction, not *expression.Column"); + errors.add("expected integer"); + errors.add("Communications link failure"); + errors.add("No operations allowed after connection closed."); + errors.add("ON condition doesn't support subqueries yet"); + errors.add("Can't group on"); // https://github.com/pingcap/tidb/issues/58974 + errors.addRegex(Pattern.compile("Expression\\s#\\d+\\sof\\sORDER\\sBY\\sis\\snot\\sin\\sGROUP\\sBY\\sclause\\sand\\scontains\\snonaggregated\\scolumn\\s'.*?'\\swhich\\sis\\snot\\sfunctionally\\sdependent\\son\\scolumns\\sin\\sGROUP\\sBY\\sclause;\\sthis\\sis\\sincompatible\\swith\\ssql_mode=only_full_group_by")); + } + + @Override + public void check() throws Exception { + + reproducer = null; + + joinsInExpr = null; + tablesFromOuterContext.clear(); + + useSubqueryAsFoldedExpr = useSubquery(); + useCorrelatedSubqueryAsFoldedExpr = useCorrelatedSubquery(); + + + TiDBSelect auxiliaryQuery = null; + + if (useSubqueryAsFoldedExpr) { + if (useCorrelatedSubqueryAsFoldedExpr) { + auxiliaryQuery = genSelectWithCorrelatedSubquery(); + auxiliaryQueryString = TiDBVisitor.asString(auxiliaryQuery); + auxiliaryQueryResult.putAll(selectResult); + } else { + auxiliaryQuery = genSelectExpression(null, null, null); + auxiliaryQueryString = TiDBVisitor.asString(auxiliaryQuery); + auxiliaryQueryResult = getQueryResult(auxiliaryQueryString, state); + } + } else { + auxiliaryQuery = genSimpleSelect(); + auxiliaryQueryString = TiDBVisitor.asString(auxiliaryQuery); + auxiliaryQueryResult.putAll(selectResult); + } + + + TiDBSelect originalQuery = null; + + Map> foldedResult = new HashMap<>(); + Map> originalResult = new HashMap<>(); + + // dependent expression + if (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) { + TiDBExpressionBag specificCondition = new TiDBExpressionBag(this.foldedExpr); + originalQuery = this.genSelectExpression(null, specificCondition, null); + originalQueryString = TiDBVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + specificCondition.updateInnerExpr(this.constantResOfFoldedExpr); + foldedQueryString = TiDBVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + + // independent expression + // empty result, put the inner query in (NOT) EXIST + else if (auxiliaryQueryResult.size() == 0 || auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().iterator().next()).size() == 0) { + boolean isNegated = Randomly.getBoolean() ? false : true; + + // original query + TiDBExists existExpr = new TiDBExists(auxiliaryQuery, isNegated); + TiDBExpressionBag specificCondition = new TiDBExpressionBag(existExpr); + + originalQuery = this.genSelectExpression(null, specificCondition, foldedExpressionReturnType); + originalQueryString = TiDBVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + + // folded query + TiDBExpression equivalentExpr = TiDBConstant.createBooleanConstant(isNegated); + specificCondition.updateInnerExpr(equivalentExpr); + foldedQueryString = TiDBVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + // Scalar Subquery: 1 column and 1 row, consider the inner query as a constant + else if (auxiliaryQueryResult.size() == 1 && auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]).size() == 1 && Randomly.getBoolean()) { + // original query + TiDBExpressionBag specificCondition = new TiDBExpressionBag(auxiliaryQuery); + originalQuery = this.genSelectExpression(null, specificCondition, getColumnTypeFromSelect(auxiliaryQuery).get(0)); + originalQueryString = TiDBVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + TiDBExpression equivalentExpr = auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]).get(0); + specificCondition.updateInnerExpr(equivalentExpr);; + foldedQueryString = TiDBVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + // one column + else if (auxiliaryQueryResult.size() == 1 && Randomly.getBoolean()) { + // original query + List columns = s.getRandomTableNonEmptyTables().getColumns(); + TiDBColumnReference selectedColumn = new TiDBColumnReference(Randomly.fromList(columns)); + TiDBTable selectedTable = selectedColumn.getColumn().getTable(); + TiDBTableReference selectedTableRef = new TiDBTableReference(selectedTable); + TiDBExpressionBag tableBag = new TiDBExpressionBag(selectedTableRef); + + TiDBInOperator optInOperation = new TiDBInOperator(selectedColumn, auxiliaryQuery); + TiDBExpressionBag specificCondition = new TiDBExpressionBag(optInOperation); + originalQuery = this.genSelectExpression(tableBag, specificCondition, null); + originalQueryString = TiDBVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + TiDBColumn tempColumn = new TiDBColumn("c0", getColumnTypeFromSelect(auxiliaryQuery).get(0), false, false, false); + LinkedHashMap> value = new LinkedHashMap<>(); + value.put(tempColumn, auxiliaryQueryResult.values().iterator().next()); + TiDBValues refValues = new TiDBValues(value); + TiDBInOperator refInOperation = new TiDBInOperator(selectedColumn, refValues); + specificCondition.updateInnerExpr(refInOperation); + foldedQueryString = TiDBVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + + // ALL + else if (auxiliaryQueryResult.size() == 1 && Randomly.getBoolean() && false) { + // a bug reported related to this feature + // original query + List columns = s.getRandomTableNonEmptyTables().getColumns(); + TiDBColumnReference selectedColumn = new TiDBColumnReference(Randomly.fromList(columns)); + TiDBTable selectedTable = selectedColumn.getColumn().getTable(); + TiDBTableReference selectedTableRef = new TiDBTableReference(selectedTable); + TiDBExpressionBag tableBag = new TiDBExpressionBag(selectedTableRef); + + TiDBExpressionGenerator exprGen = new TiDBExpressionGenerator(state).setColumns(Arrays.asList(selectedColumn.getColumn())); + TiDBExpression allOptLeft = genCondition(exprGen, null, null); + TiDBComparisonOperator allOperator = TiDBComparisonOperator.getRandom(); + while (allOperator == TiDBComparisonOperator.NULL_SAFE_EQUALS) { + allOperator = TiDBComparisonOperator.getRandom(); + } + TiDBAllOperator optAllOperation = new TiDBAllOperator(allOptLeft, auxiliaryQuery, allOperator); + TiDBExpressionBag specificCondition = new TiDBExpressionBag(optAllOperation); + originalQuery = this.genSelectExpression(tableBag, specificCondition, null); + originalQueryString = TiDBVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + TiDBColumn tempColumn = new TiDBColumn("c0", getColumnTypeFromSelect(auxiliaryQuery).get(0), false, false, false); + LinkedHashMap> value = new LinkedHashMap<>(); + value.put(tempColumn, auxiliaryQueryResult.values().iterator().next()); + TiDBValues refValues = new TiDBValues(value); + TiDBAllOperator refAllOperation = new TiDBAllOperator(allOptLeft, refValues, allOperator); + specificCondition.updateInnerExpr(refAllOperation); + foldedQueryString = TiDBVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + + // ANY + else if (auxiliaryQueryResult.size() == 1 && Randomly.getBoolean()) { + // original query + List columns = s.getRandomTableNonEmptyTables().getColumns(); + TiDBColumnReference selectedColumn = new TiDBColumnReference(Randomly.fromList(columns)); + TiDBTable selectedTable = selectedColumn.getColumn().getTable(); + TiDBTableReference selectedTableRef = new TiDBTableReference(selectedTable); + TiDBExpressionBag tableBag = new TiDBExpressionBag(selectedTableRef); + + TiDBExpressionGenerator exprGen = new TiDBExpressionGenerator(state).setColumns(Arrays.asList(selectedColumn.getColumn())); + TiDBExpression anyOptLeft = genCondition(exprGen, null, null); + TiDBComparisonOperator anyOperator = TiDBComparisonOperator.getRandom(); + while (anyOperator == TiDBComparisonOperator.NULL_SAFE_EQUALS) { + anyOperator = TiDBComparisonOperator.getRandom(); + } + TiDBAnyOperator optAnyOperation = new TiDBAnyOperator(anyOptLeft, auxiliaryQuery, anyOperator); + TiDBExpressionBag specificCondition = new TiDBExpressionBag(optAnyOperation); + originalQuery = this.genSelectExpression(tableBag, specificCondition, null); + originalQueryString = TiDBVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + TiDBColumn tempColumn = new TiDBColumn("c0", getColumnTypeFromSelect(auxiliaryQuery).get(0), false, false, false); + LinkedHashMap> value = new LinkedHashMap<>(); + value.put(tempColumn, auxiliaryQueryResult.values().iterator().next()); + TiDBValues refValues = new TiDBValues(value); + TiDBAnyOperator refAnyOperation = new TiDBAnyOperator(anyOptLeft, refValues, anyOperator); + specificCondition.updateInnerExpr(refAnyOperation); + foldedQueryString = TiDBVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + + // Row Subquery + else { + // original query + TiDBTable temporaryTable = this.genTemporaryTable(auxiliaryQuery, tempTableName); + TiDBTableReference tempTableRef = new TiDBTableReference(temporaryTable); + + LinkedHashMap> value = new LinkedHashMap<>(); + for (TiDBColumn c: temporaryTable.getColumns()) { + value.put(c, auxiliaryQueryResult.get(c.getName())); + } + TiDBValuesRow resValues = new TiDBValuesRow(value); + + TiDBExpressionBag tempTableRefBag = new TiDBExpressionBag(tempTableRef); + TiDBTableAndColumnReference tableAndColumnRef = new TiDBTableAndColumnReference(temporaryTable); + TiDBWithClasure withClasure = null; + if (Randomly.getBoolean() || true) { + withClasure = new TiDBWithClasure(tableAndColumnRef, auxiliaryQuery); + } else { + // there is an error in `WITH t0(c0) AS VALUES` + withClasure = new TiDBWithClasure(tableAndColumnRef, resValues); + } + originalQuery = genSelectExpression(tempTableRefBag, null, null); + originalQuery.setWithClause(withClasure); + originalQueryString = TiDBVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + if (Randomly.getBoolean() && false) { + // folded query: FROM VALUES () AS table, tidb seems not support this + originalQuery.setWithClause(null); + TiDBAlias alias = new TiDBAlias(resValues, TiDBVisitor.asString(tableAndColumnRef)); + tempTableRefBag.updateInnerExpr(alias); + foldedQueryString = TiDBVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } else if (Randomly.getBoolean()) { + // folded query: SELECT FROM () AS table + originalQuery.setWithClause(null); + TiDBAlias alias = null; + if (Randomly.getBoolean() || true) { + alias = new TiDBAlias(auxiliaryQuery, TiDBVisitor.asString(tempTableRef)); + } else { + // SELECT * FROM (VALUES ROW(1)) AS t2(c0); is not supported in TiDB + alias = new TiDBAlias(resValues, TiDBVisitor.asString(tableAndColumnRef)); + } + tempTableRefBag.updateInnerExpr(alias); + foldedQueryString = TiDBVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } else { + // folded query: CREATE the table + try { + this.createTemporaryTable(auxiliaryQuery, tempTableName, TiDBVisitor.asString(resValues)); + originalQuery.setWithClause(null); + foldedQueryString = TiDBVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } finally { + dropTemporaryTable(tempTableName); + } + } + } + + if (foldedResult == null || originalResult == null) { + throw new IgnoreMeException(); + } + if (foldedQueryString.equals(originalQueryString)) { + throw new IgnoreMeException(); + } + if (!compareResult(foldedResult, originalResult)) { + reproducer = null; // TODO + state.getState().getLocalState().log(auxiliaryQueryString + ";\n" +foldedQueryString + ";\n" + originalQueryString + ";"); + throw new AssertionError(auxiliaryQueryResult.toString() + " " +foldedResult.toString() + " " + originalResult.toString()); + } + } + + private TiDBSelect genSelectExpression(TiDBExpressionBag tableBag, TiDBExpression specificCondition, TiDBCompositeDataType conditionType) { + TiDBTables randomTables = s.getRandomTableNonEmptyTables(); + if (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) { + for (TiDBTable t : this.tablesFromOuterContext) { + randomTables.addTable(t); + } + if (this.joinsInExpr != null) { + for (TiDBJoin j : this.joinsInExpr) { + TiDBTableReference lt = (TiDBTableReference) j.getLeftTable(); + randomTables.removeTable(lt.getTable()); + TiDBTableReference rt = (TiDBTableReference) j.getRightTable(); + randomTables.removeTable(rt.getTable()); + } + } + } + TiDBTable tempTable = null; + TiDBTableReference tableRef = null; + if (tableBag != null) { + tableRef = (TiDBTableReference) tableBag.getInnerExpr(); + tempTable = tableRef.getTable(); + } + List columns = randomTables.getColumns(); + if (tempTable != null) { + columns.addAll(tempTable.getColumns()); + } + if ((!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) && this.joinsInExpr != null) { + for (TiDBJoin j : this.joinsInExpr) { + TiDBTable t = ((TiDBTableReference) j.getRightTable()).getTable(); + columns.addAll(t.getColumns()); + t = ((TiDBTableReference) j.getLeftTable()).getTable(); + columns.addAll(t.getColumns()); + } + } + gen = new TiDBExpressionGenerator(state).setColumns(columns); + List tables = randomTables.getTables(); + List tableRefs = tables.stream().map(t -> new TiDBTableReference(t)).collect(Collectors.toList()); + + TiDBSelect select = new TiDBSelect(); + + // TiDB currently not support subquery in ON + // List joins = genJoinExpressions(tableRefs, state, specificCondition, conditionType); + List joins = new ArrayList<>(); + if ((!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr))) { + if (this.joinsInExpr != null) { + joins.addAll(this.joinsInExpr); + this.joinsInExpr = null; + } + } + else if (Randomly.getBoolean()) { + joins = genJoinExpressions(tableRefs, state, specificCondition, conditionType); + } + if (joins.size() > 0) { + select.setJoinClauses(joins); + } + + if (tableBag != null) { + TiDBTableReference outerTable = (TiDBTableReference) tableBag.getInnerExpr(); + boolean isContained = false; + for (TiDBExpression e: tableRefs) { + TiDBTableReference tr = (TiDBTableReference) e; + if (tr.getTable().getName().equals(outerTable.getTable().getName())) { + isContained = true; + } + } + if (joins.size() > 0) { + for (TiDBJoin j : joins) { + TiDBTable t = ((TiDBTableReference) j.getRightTable()).getTable(); + if (t.getName().equals(outerTable.getTable().getName())) { + isContained = true; + } + t = ((TiDBTableReference) j.getLeftTable()).getTable(); + if (t.getName().equals(outerTable.getTable().getName())) { + isContained = true; + } + } + } + if (!isContained) { + tableRefs.add(tableBag); + } + } + + select.setFromList(tableRefs); + + select.setWhereClause(genCondition(gen, specificCondition, conditionType)); + + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(genOrderBys(gen, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null, conditionType)); + } + + if (Randomly.getBoolean()) { + List selectedColumns = Randomly.nonEmptySubset(columns); + List selectedAlias = new LinkedList<>(); + for (int i = 0; i < selectedColumns.size(); ++i) { + TiDBColumnReference originalName = new TiDBColumnReference(selectedColumns.get(i)); + TiDBAlias columnAlias = new TiDBAlias(originalName, "c" + String.valueOf(i)); + selectedAlias.add(columnAlias); + } + select.setFetchColumns(selectedAlias); + } else { + TiDBColumn selectedColumn = Randomly.fromList(columns); + TiDBColumnReference aggr = new TiDBColumnReference(selectedColumn); + TiDBAggregateFunction windowFunction = TiDBAggregateFunction.getRandom(); + // one bug reported about this + while(windowFunction == TiDBAggregateFunction.BIT_AND || windowFunction == TiDBAggregateFunction.BIT_OR) { + windowFunction = TiDBAggregateFunction.getRandom(); + } + TiDBExpression originalName = new TiDBAggregate(Arrays.asList(aggr), windowFunction); + TiDBAlias columnAlias = new TiDBAlias(originalName, "c0"); + select.setFetchColumns(Arrays.asList(columnAlias)); + // there are many syntax error in group by with subquery, just remove it + select.setGroupByExpressions(genGroupBys(Randomly.nonEmptySubset(columns), Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null, conditionType)); + select.setGroupByExpressions(gen.generateGroupBys()); + + // gen having + // there is an error in having has not been fixed: unknown column in having clause + if (Randomly.getBooleanWithRatherLowProbability()) { + TiDBExpressionGenerator havingGen = new TiDBExpressionGenerator(state).setColumns(columns); + select.setHavingClause(genCondition(havingGen, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null, conditionType)); + } + } + return select; + } + + private TiDBExpression genCondition(TiDBExpressionGenerator conGen, TiDBExpression specificCondition, TiDBCompositeDataType conditionType) { + TiDBExpression randomCondition = conGen.generateBooleanExpression(); + if (specificCondition != null) { + if (conditionType == null) { + randomCondition = new TiDBBinaryLogicalOperation(randomCondition, specificCondition, TiDBBinaryLogicalOperator.getRandom()); + } else { + switch(conditionType.getPrimitiveDataType()) { + case BOOL: + randomCondition = new TiDBBinaryLogicalOperation(randomCondition, specificCondition, TiDBBinaryLogicalOperator.getRandom()); + break; + + case DECIMAL: + case FLOATING: + case INT: + case TEXT: + case CHAR: + case NUMERIC: + case BLOB: + randomCondition = new TiDBBinaryComparisonOperation(randomCondition, specificCondition, TiDBComparisonOperator.getRandom()); + break; + default: + randomCondition = new TiDBBinaryLogicalOperation(randomCondition, specificCondition, TiDBBinaryLogicalOperator.getRandom()); + break; + } + } + } + return randomCondition; + } + + private List genJoinExpressions(List tableList, TiDBGlobalState globalState, TiDBExpression specificCondition, TiDBCompositeDataType conditionType) { + List joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBoolean()) { + TiDBTableReference leftTable = (TiDBTableReference) tableList.remove(0); + TiDBTableReference rightTable = (TiDBTableReference) tableList.remove(0); + List columns = new ArrayList<>(leftTable.getTable().getColumns()); + columns.addAll(rightTable.getTable().getColumns()); + TiDBExpressionGenerator joinGen = new TiDBExpressionGenerator(globalState).setColumns(columns); + TiDBExpression randomCondition = genCondition(joinGen, specificCondition, conditionType); + switch (TiDBJoin.JoinType.getRandom()) { + case INNER: + joinExpressions.add(TiDBJoin.createInnerJoin(leftTable, rightTable, randomCondition)); + break; + case NATURAL: + joinExpressions.add(TiDBJoin.createNaturalJoin(leftTable, rightTable, NaturalJoinType.getRandom())); + break; + case STRAIGHT: + joinExpressions.add(TiDBJoin.createStraightJoin(leftTable, rightTable, randomCondition)); + break; + case LEFT: + joinExpressions.add(TiDBJoin.createLeftOuterJoin(leftTable, rightTable, randomCondition)); + break; + case RIGHT: + joinExpressions.add(TiDBJoin.createRightOuterJoin(leftTable, rightTable, randomCondition)); + break; + case CROSS: + joinExpressions.add(TiDBJoin.createCrossJoin(leftTable, rightTable, randomCondition)); + break; + default: + throw new AssertionError(); + } + } + return joinExpressions; + } + + public List genOrderBys(TiDBExpressionGenerator orderByGen, TiDBExpression specificCondition, TiDBCompositeDataType conditionType) { + int exprNum = Randomly.smallNumber() + 1; + List newExpressions = new ArrayList<>(); + for (int i = 0; i < exprNum; ++i) { + TiDBExpression condition = genCondition(orderByGen, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null, conditionType); + if (Randomly.getBoolean()) { + condition = new TiDBOrderingTerm(condition, Randomly.getBoolean()); + } + newExpressions.add(condition); + } + return newExpressions; + } + + private List genGroupBys(List columns, TiDBExpression specificCondition, TiDBCompositeDataType conditionType) { + TiDBExpressionGenerator groupByGen = new TiDBExpressionGenerator(state).setColumns(columns); + int exprNum = Randomly.smallNumber() + 1; + List newExpressions = new ArrayList<>(); + for (int i = 0; i < exprNum; ++i) { + TiDBExpression condition = genCondition(groupByGen, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null, conditionType); + newExpressions.add(condition); + } + return newExpressions; + } + + private Map> getQueryResult(String queryString, TiDBGlobalState state) throws SQLException { + Map> result = new LinkedHashMap<>(); + if (options.logEachSelect()) { + logger.writeCurrent(queryString); + } + Statement s = null; + try { + s = this.con.createStatement(); + ResultSet rs = null; + try { + rs = s.executeQuery(queryString); + ResultSetMetaData metaData = rs.getMetaData(); + Integer columnCount = metaData.getColumnCount(); + Map idxNameMap = new HashMap<>(); + for (int i = 1; i <= columnCount; i++) { + result.put("c" + String.valueOf(i-1), new ArrayList<>()); + idxNameMap.put(i, "c" + String.valueOf(i-1)); + } + + int resultRows = 0; + while (rs.next()) { + for (int i = 1; i <= columnCount; i++) { + try { + Object value = rs.getObject(i); + TiDBConstant constant; + if (rs.wasNull()) { + constant = TiDBConstant.createNullConstant(); + } + + else if (value instanceof Integer) { + constant = TiDBConstant.createIntConstant(BigInteger.valueOf((Integer) value)); + } else if (value instanceof Short) { + constant = TiDBConstant.createIntConstant(BigInteger.valueOf((Short) value)); + } else if (value instanceof Long) { + constant = TiDBConstant.createIntConstant(BigInteger.valueOf((Long) value)); + } else if (value instanceof BigInteger) { + constant = TiDBConstant.createIntConstant((BigInteger) value); + } + + else if (value instanceof Float) { + constant = TiDBConstant.createFloatConstant(Double.valueOf((Float) value)); + } else if (value instanceof Double) { + constant = TiDBConstant.createFloatConstant((Double) value); + } else if (value instanceof BigDecimal) { + constant = TiDBConstant.createFloatConstant(((BigDecimal) value).doubleValue()); + } + + else if (value instanceof Boolean) { + constant = TiDBConstant.createBooleanConstant((Boolean) value); + } + + else if (value instanceof String) { + constant = TiDBConstant.createStringConstant((String) value); + } + + // else if (value instanceof byte[]) { + // constant = TiDBConstant.createBitConstant(Long.parseLong(new String((byte[]) value, StandardCharsets.UTF_8), 2)); + // } + + else if (value == null) { + constant = TiDBConstant.createNullConstant(); + } else { + throw new IgnoreMeException(); + } + List v = result.get(idxNameMap.get(i)); + v.add(constant); + } catch (SQLException e) { + System.out.println(e.getMessage()); + throw new IgnoreMeException(); + } catch (NumberFormatException e) { + throw new IgnoreMeException(); + } + } + ++resultRows; + if (resultRows > 100) { + throw new IgnoreMeException(); + } + } + rs.close(); + Main.nrSuccessfulActions.addAndGet(1); + } finally { + if (rs != null) { + rs.close(); + } + } + } catch (SQLException e) { + Main.nrUnsuccessfulActions.addAndGet(1); + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + state.getState().getLocalState().log(queryString); + throw new AssertionError(e.getMessage()); + } + } finally { + if (s != null) { + s.close(); + } + } + return result; + } + + private TiDBTable genTemporaryTable(TiDBSelect select, String tableName) { + List fetchColumns = select.getFetchColumns(); + int columnNumber = fetchColumns.size(); + Map idxTypeMap = getColumnTypeFromSelect(select); + + List databaseColumns = new ArrayList<>(); + for (int i = 0; i < columnNumber; ++i) { + String columnName = "c" + String.valueOf(i); + TiDBColumn column = new TiDBColumn(columnName, idxTypeMap.get(i), false, false, false); + databaseColumns.add(column); + } + TiDBTable table = new TiDBTable(tableName, databaseColumns, null, false); + for (TiDBColumn c : databaseColumns) { + c.setTable(table); + } + + return table; + } + + private TiDBTable createTemporaryTable(TiDBSelect select, String tableName, String valuesString) throws SQLException { + List fetchColumns = select.getFetchColumns(); + int columnNumber = fetchColumns.size(); + Map idxTypeMap = getColumnTypeFromSelect(select); + + StringBuilder sb = new StringBuilder(); + sb.append("CREATE TABLE " + tableName + " ("); + for (int i = 0; i < columnNumber; ++i) { + String columnTypeName = ""; + if (idxTypeMap.get(i) != null) { + columnTypeName = idxTypeMap.get(i).getPrimitiveDataType().name(); + } + sb.append("c" + String.valueOf(i) + " " + columnTypeName + ", "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append(");"); + String crateTableString = sb.toString(); + if (options.logEachSelect()) { + logger.writeCurrent(crateTableString); + } + Statement s = null; + try { + s = this.con.createStatement(); + try { + s.execute(crateTableString); + } catch (SQLException e) { + throw new IgnoreMeException(); + } + } finally { + if (s != null) { + s.close(); + } + } + + String selectString = TiDBVisitor.asString(select); + StringBuilder sb2 = new StringBuilder(); + if (Randomly.getBoolean()) { + sb2.append("INSERT INTO " + tableName + " "+ selectString); + } else { + sb2.append("INSERT INTO " + tableName + " "+ valuesString); + } + + String insertValueString = sb2.toString(); + if (options.logEachSelect()) { + logger.writeCurrent(insertValueString); + } + s = null; + try { + s = this.con.createStatement(); + try { + s.execute(insertValueString); + } catch (SQLException e) { + throw new IgnoreMeException(); + } + } finally { + s.close(); + } + + List databaseColumns = new ArrayList<>(); + for (int i = 0; i < columnNumber; ++i) { + String columnName = "c" + String.valueOf(i); + TiDBColumn column = new TiDBColumn(columnName, idxTypeMap.get(i), false, false, false); + databaseColumns.add(column); + } + TiDBTable table = new TiDBTable(tableName, databaseColumns, null, false); + for (TiDBColumn c : databaseColumns) { + c.setTable(table); + } + + return table; + } + + private void dropTemporaryTable(String tableName) throws SQLException { + String dropString = "DROP TABLE " + tableName + ";"; + if (options.logEachSelect()) { + logger.writeCurrent(dropString); + } + Statement s = null; + try { + s = this.con.createStatement(); + try { + s.execute(dropString); + } catch (SQLException e) { + throw new IgnoreMeException(); + } + } finally { + s.close(); + } + } + + private Map getColumnTypeFromSelect(TiDBSelect select) { + List fetchColumns = select.getFetchColumns(); + Map idxTypeMap = new HashMap<>(); + for (int i = 0; i < fetchColumns.size(); ++i) { + TiDBExpression column = fetchColumns.get(i); + TiDBCompositeDataType columnType = null; + if (column instanceof TiDBColumnReference) { + TiDBColumnReference c = (TiDBColumnReference) column; + columnType = c.getColumn().getType(); + } else if (column instanceof TiDBAlias) { + TiDBAlias a = (TiDBAlias) column; + TiDBExpression left = a.getExpression(); + if (left instanceof TiDBColumnReference) { + TiDBColumnReference c = (TiDBColumnReference) left; + columnType = c.getColumn().getType(); + } else if (left instanceof TiDBAggregate) { + // TiDBAggregate aggr = (TiDBAggregate) left; + // List aggrExprs = aggr.getArgs(); + // TiDBExpression aggrExpr = aggrExprs.get(0); + // if (aggrExpr instanceof TiDBColumnReference) { + // TiDBColumnReference c = (TiDBColumnReference) aggrExpr; + // columnType = c.getColumn().getType(); + // } else { + // throw new IgnoreMeException(); + // } + columnType = new TiDBCompositeDataType(TiDBDataType.INT, 8); + } + } + if (columnType == null) { + columnType = TiDBCompositeDataType.getRandom(); + } + idxTypeMap.put(i, columnType); + } + + return idxTypeMap; + } + + private boolean compareResult(Map> r1, Map> r2) { + if (r1.size() != r2.size()) { + return false; + } + for (Map.Entry < String, List > entry: r1.entrySet()) { + String currentKey = entry.getKey(); + if (!r2.containsKey(currentKey)) { + return false; + } + List v1= entry.getValue(); + List v2= r2.get(currentKey); + if (v1.size() != v2.size()) { + return false; + } + List v1Value = new ArrayList<>(v1.stream().map(c -> c.toString()).collect(Collectors.toList())); + List v2Value = new ArrayList<>(v2.stream().map(c -> c.toString()).collect(Collectors.toList())); + Collections.sort(v1Value); + Collections.sort(v2Value); + if (!v1Value.equals(v2Value)) { + return false; + } + } + return true; + } + + private TiDBSelect genSimpleSelect() { + TiDBTables tables = s.getRandomTableNonEmptyTables(); + tablesFromOuterContext = tables.getTables(); + List tableL = tables.getTables().stream().map(t -> new TiDBTableReference(t)) + .collect(Collectors.toList()); + TiDBExpressionGenerator exprGen = new TiDBExpressionGenerator(state).setColumns(tables.getColumns()); + this.foldedExpr = genCondition(exprGen, null, null); + + TiDBSelect select = new TiDBSelect(); + if (Randomly.getBoolean()) { + List joins = genJoinExpressions(tableL, state, null, null); + if (joins.size() > 0) { + select.setJoinClauses(joins); + this.joinsInExpr = joins; + } + } + select.setFromList(tableL); + + List fetchColumns = new ArrayList<>(); + int columnIdx = 0; + for (TiDBColumn c : tables.getColumns()) { + TiDBColumnReference cRef = new TiDBColumnReference(c); + TiDBAlias cAlias = new TiDBAlias(cRef, "c" + String.valueOf(columnIdx)); + fetchColumns.add(cAlias); + columnIdx++; + } + + // add the expression as last fetch column + TiDBAlias eAlias = new TiDBAlias(this.foldedExpr, "c" + String.valueOf(columnIdx)); + fetchColumns.add(eAlias); + + select.setFetchColumns(fetchColumns); + + originalQueryString = TiDBVisitor.asString(select); + + + Map> queryRes = null; + try { + queryRes = getQueryResult(originalQueryString, state); + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(e.getMessage()); + } + } + if (queryRes.get("c0").size() == 0) { + throw new IgnoreMeException(); + } + + // save the result first + selectResult.clear(); + selectResult.putAll(queryRes); + + // get the constant corresponding to each row from results + List summary = queryRes.remove("c" + String.valueOf(columnIdx)); + + Boolean emptyRes = queryRes.get(queryRes.keySet().iterator().next()).size() == 0; + + Map columnType = null; + if (!emptyRes) { + columnType = getColumnTypeFromSelect(select); + } + LinkedHashMap> dbstate = new LinkedHashMap<>(); + // do not put the last fetch column to values + for (int i = 0; i < fetchColumns.size() - 1; ++i) { + TiDBAlias cAlias = (TiDBAlias) fetchColumns.get(i); + TiDBColumnReference cRef = (TiDBColumnReference) cAlias.getExpression(); + String columnName = cAlias.getAlias(); + dbstate.put(cRef, queryRes.get(columnName)); + } + + foldedExpressionReturnType = columnType.get(fetchColumns.size() - 1); + + this.constantResOfFoldedExpr = new TiDBResultMap(dbstate, summary, foldedExpressionReturnType); + + return select; + } + + private TiDBSelect genSelectWithCorrelatedSubquery() { + TiDBTables outerQueryRandomTables = s.getRandomTableNonEmptyTables(); + TiDBTables innerQueryRandomTables = s.getRandomTableNonEmptyTables(); + + List innerQueryFromTables = new ArrayList<>(); + for (TiDBTable t : innerQueryRandomTables.getTables()) { + if (!outerQueryRandomTables.isContained(t)) { + innerQueryFromTables.add(new TiDBTableReference(t)); + } + } + for (TiDBTable t : outerQueryRandomTables.getTables()) { + if (innerQueryRandomTables.isContained(t)) { + innerQueryRandomTables.removeTable(t); + + List newColumns = new ArrayList<>(); + for (TiDBColumn c : t.getColumns()) { + TiDBColumn newColumn = new TiDBColumn(c.getName(), c.getType(), false, false, false); + newColumns.add(newColumn); + } + TiDBTable newTable = new TiDBTable(t.getName() + "a", newColumns, null, false); + for (TiDBColumn c : newColumns) { + c.setTable(newTable); + } + innerQueryRandomTables.addTable(newTable); + + TiDBAlias alias = new TiDBAlias(new TiDBTableReference(t), newTable.getName()); + innerQueryFromTables.add(alias); + } + } + + List innerQueryColumns = new ArrayList<>(); + innerQueryColumns.addAll(innerQueryRandomTables.getColumns()); + innerQueryColumns.addAll(outerQueryRandomTables.getColumns()); + gen = new TiDBExpressionGenerator(state).setColumns(innerQueryColumns); + + TiDBSelect innerQuery = new TiDBSelect(); + innerQuery.setFromList(innerQueryFromTables); + + TiDBExpression innerQueryWhereCondition = gen.generateBooleanExpression(); + innerQuery.setWhereClause(innerQueryWhereCondition); + + // use aggregate function in fetch column + TiDBColumnReference innerQueryAggr = new TiDBColumnReference(Randomly.fromList(innerQueryRandomTables.getColumns())); + TiDBAggregateFunction windowFunction = TiDBAggregateFunction.getRandom(); + TiDBExpression innerQueryAggrName = new TiDBAggregate(Arrays.asList(innerQueryAggr), windowFunction); + innerQuery.setFetchColumns(Arrays.asList(innerQueryAggrName)); + + this.foldedExpr = innerQuery; + + // outer query + TiDBSelect outerQuery = new TiDBSelect(); + List outerQueryFromTableRefs = outerQueryRandomTables.getTables().stream().map(t -> new TiDBTableReference(t)).collect(Collectors.toList()); + outerQuery.setFromList(outerQueryFromTableRefs); + tablesFromOuterContext = outerQueryRandomTables.getTables(); + + List fetchColumns = new ArrayList<>(); + int columnIdx = 0; + for (TiDBColumn c : outerQueryRandomTables.getColumns()) { + TiDBColumnReference cRef = new TiDBColumnReference(c); + TiDBAlias cAlias = new TiDBAlias(cRef, "c" + String.valueOf(columnIdx)); + fetchColumns.add(cAlias); + columnIdx++; + } + + // add the expression as last fetch column + TiDBAlias subqueryAlias = new TiDBAlias(innerQuery, "c" + String.valueOf(columnIdx)); + fetchColumns.add(subqueryAlias); + + outerQuery.setFetchColumns(fetchColumns); + + originalQueryString = TiDBVisitor.asString(outerQuery); + + + Map> queryRes = null; + try { + queryRes = getQueryResult(originalQueryString, state); + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(e.getMessage()); + } + } + // just ignore the empty result, because of the empty table + if (queryRes.get("c0").size() == 0) { + throw new IgnoreMeException(); + } + + // save the result first + selectResult.clear(); + selectResult.putAll(queryRes); + + // get the constant corresponding to each row from results + List summary = queryRes.remove("c" + String.valueOf(columnIdx)); + + Boolean emptyRes = queryRes.get(queryRes.keySet().iterator().next()).size() == 0; + Map columnType = null; + if (!emptyRes) { + columnType = getColumnTypeFromSelect(outerQuery); + } + + LinkedHashMap> dbstate = new LinkedHashMap<>(); + // do not put the last fetch column to values + for (int i = 0; i < fetchColumns.size() - 1; ++i) { + TiDBAlias cAlias = (TiDBAlias) fetchColumns.get(i); + TiDBColumnReference cRef = (TiDBColumnReference) cAlias.getExpression(); + String columnName = cAlias.getAlias(); + dbstate.put(cRef, queryRes.get(columnName)); + } + + foldedExpressionReturnType = columnType.get(fetchColumns.size() - 1); + + this.constantResOfFoldedExpr = new TiDBResultMap(dbstate, summary, foldedExpressionReturnType); + + return outerQuery; + } + + Boolean isEmptyTable(TiDBTable t) throws SQLException { + String queryString = "SELECT * FROM " + TiDBVisitor.asString(new TiDBTableReference(t)) + ";"; + int resultRows = 0; + Statement s = null; + try { + s = this.con.createStatement(); + ResultSet rs = null; + try { + rs = s.executeQuery(queryString); + while (rs.next()) { + ++resultRows; + } + rs.close(); + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + state.getState().getLocalState().log(queryString); + throw new AssertionError(e.getMessage()); + } + } finally { + rs.close(); + } + } finally { + if (s != null) { + s.close(); + } + } + return resultRows == 0; + } + + public boolean useSubquery() { + if (this.state.getDbmsSpecificOptions().coddTestModel.equals("random")) { + return Randomly.getBoolean(); + } else if (this.state.getDbmsSpecificOptions().coddTestModel.equals("expression")) { + return false; + } else if (this.state.getDbmsSpecificOptions().coddTestModel.equals("subquery")) { + return true; + } else { + System.out.printf("Wrong option of --coddtest-model, should be one of: random, expression, subquery"); + System.exit(1); + return false; + } + } + + public boolean useCorrelatedSubquery() { + return Randomly.getBoolean(); + } + + @Override + public String getLastQueryString() { + return originalQueryString; + } + + @Override + public Reproducer getLastReproducer() { + return reproducer; + } +} diff --git a/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java b/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java index 67c7cda31..ad1d762e0 100644 --- a/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java +++ b/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java @@ -1,19 +1,38 @@ package sqlancer.tidb.visitor; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.stream.Collectors; + import sqlancer.Randomly; import sqlancer.common.visitor.ToStringVisitor; +import sqlancer.tidb.TiDBSchema.TiDBColumn; +import sqlancer.tidb.TiDBSchema.TiDBCompositeDataType; +import sqlancer.tidb.TiDBSchema.TiDBTable; import sqlancer.tidb.ast.TiDBAggregate; +import sqlancer.tidb.ast.TiDBAlias; +import sqlancer.tidb.ast.TiDBAllOperator; +import sqlancer.tidb.ast.TiDBAnyOperator; import sqlancer.tidb.ast.TiDBCase; import sqlancer.tidb.ast.TiDBCastOperation; import sqlancer.tidb.ast.TiDBColumnReference; import sqlancer.tidb.ast.TiDBConstant; +import sqlancer.tidb.ast.TiDBConstant.TiDBNullConstant; +import sqlancer.tidb.ast.TiDBExists; import sqlancer.tidb.ast.TiDBExpression; +import sqlancer.tidb.ast.TiDBExpressionBag; import sqlancer.tidb.ast.TiDBFunctionCall; +import sqlancer.tidb.ast.TiDBInOperator; import sqlancer.tidb.ast.TiDBJoin; +import sqlancer.tidb.ast.TiDBResultMap; import sqlancer.tidb.ast.TiDBJoin.JoinType; import sqlancer.tidb.ast.TiDBSelect; +import sqlancer.tidb.ast.TiDBTableAndColumnReference; import sqlancer.tidb.ast.TiDBTableReference; import sqlancer.tidb.ast.TiDBText; +import sqlancer.tidb.ast.TiDBValues; +import sqlancer.tidb.ast.TiDBValuesRow; +import sqlancer.tidb.ast.TiDBWithClasure; public class TiDBToStringVisitor extends ToStringVisitor implements TiDBVisitor { @@ -47,6 +66,10 @@ public void visit(TiDBTableReference expr) { @Override public void visit(TiDBSelect select) { + if (select.getWithClause() != null) { + visit(select.getWithClause()); + sb.append(" "); + } sb.append("SELECT "); if (select.getHint() != null) { sb.append("/*+ "); @@ -182,4 +205,216 @@ public void visit(TiDBCase op) { } sb.append(" END )"); } + + + // CODDTest + @Override + public void visit(TiDBAlias alias) { + TiDBExpression e = alias.getExpression(); + if (e instanceof TiDBSelect) { + sb.append("("); + } + visit(e); + if (e instanceof TiDBSelect) { + sb.append(")"); + } + sb.append(" AS "); + sb.append(alias.getAlias()); + } + + @Override + public void visit(TiDBExists exists) { + if (exists.getNegated()) { + sb.append(" NOT"); + } + sb.append(" EXISTS("); + visit(exists.getExpression()); + sb.append(")"); + } + + @Override + public void visit(TiDBExpressionBag exprBag) { + visit(exprBag.getInnerExpr()); + } + + @Override + public void visit(TiDBInOperator inOperation) { + sb.append(" "); + visit(inOperation.getLeft()); + + sb.append(" IN "); + sb.append("("); + visit(inOperation.getRight()); + sb.append(")"); + } + @Override + public void visit(TiDBTableAndColumnReference tAndCRef) { + TiDBTable table = tAndCRef.getTable(); + sb.append(table.getName()); + sb.append("("); + sb.append(table.getColumnsAsString()); + sb.append(") "); + } + + @Override + public void visit(TiDBValues values) { + LinkedHashMap> vs = values.getValues(); + int size = vs.get(vs.keySet().iterator().next()).size(); + // sb.append("VALUES "); + // sb.append("("); + for (int i = 0; i < size; i++) { + sb.append("("); + for (TiDBColumn name : vs.keySet()) { + visit(vs.get(name).get(i)); + sb.append(", "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append("), "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + } + + @Override + public void visit(TiDBValuesRow values) { + // https://database.guide/values-statement-in-mysql/ + LinkedHashMap> vs = values.getValues(); + int size = vs.get(vs.keySet().iterator().next()).size(); + sb.append("(VALUES "); + for (int i = 0; i < size; i++) { + sb.append("ROW("); + for (TiDBColumn name : vs.keySet()) { + visit(vs.get(name).get(i)); + sb.append(", "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append("), "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append(")"); + } + + @Override + public void visit(TiDBWithClasure withClasure) { + sb.append("WITH "); + visit(withClasure.getLeft()); + sb.append(" AS ("); + visit(withClasure.getRight()); + sb.append(") "); + } + + @Override + public void visit(TiDBResultMap tableSummary) { + // we use CASE WHEN THEN END here + LinkedHashMap> vs = tableSummary.getDbStates(); + List results = tableSummary.getResult(); + + int size = vs.get(vs.keySet().iterator().next()).size(); + if (size == 0) { + sb.append("("); + for (TiDBColumnReference tr: vs.keySet()) { + visit(tr); + sb.append(" IS NULL AND "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append(")"); + return; + } + + sb.append(" CASE "); + for (int i = 0; i < size; i++) { + sb.append("WHEN "); + for (TiDBColumnReference tr: vs.keySet()) { + visit(tr); + if (vs.get(tr).get(i) instanceof TiDBNullConstant) { + sb.append(" IS NULL"); + } else { + sb.append(" = "); + sb.append(vs.get(tr).get(i).toString()); + // if (values.getColumns().get(j).getType() != null) { + // sb.append("::" + values.getColumns().get(j).getType().toString()); + // } + } + sb.append(" AND "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append("THEN "); + visit(results.get(i)); + // sb.append("::" + summaryDataType.toString() + " "); + sb.append(" "); + } + sb.append("END "); + } + + @Override + public void visit(TiDBAllOperator allOperation) { + sb.append("("); + visit(allOperation.getLeftExpr()); + sb.append(") "); + sb.append(allOperation.getOperator()); + sb.append(" ALL ("); + if (allOperation.getRightExpr() instanceof TiDBValues) { + TiDBValues values = (TiDBValues) allOperation.getRightExpr(); + LinkedHashMap> vs = values.getValues(); + int size = vs.get(vs.keySet().iterator().next()).size(); + for (int i = 0; i < size; i++) { + if (i != 0) { + sb.append(" UNION SELECT "); + } else { + sb.append(" SELECT "); + } + + for (TiDBColumn name : vs.keySet()) { + visit(vs.get(name).get(i)); + sb.append(", "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + } + } else { + visit(allOperation.getRightExpr()); + } + sb.append(")"); + } + + @Override + public void visit(TiDBAnyOperator anyOperation) { + sb.append("("); + visit(anyOperation.getLeftExpr()); + sb.append(") "); + sb.append(anyOperation.getOperator()); + sb.append(" ANY ("); + if (anyOperation.getRightExpr() instanceof TiDBValues) { + TiDBValues values = (TiDBValues) anyOperation.getRightExpr(); + LinkedHashMap> vs = values.getValues(); + int size = vs.get(vs.keySet().iterator().next()).size(); + for (int i = 0; i < size; i++) { + if (i != 0) { + sb.append(" UNION SELECT "); + } else { + sb.append(" SELECT "); + } + + for (TiDBColumn name : vs.keySet()) { + visit(vs.get(name).get(i)); + sb.append(", "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + } + } else { + visit(anyOperation.getRightExpr()); + } + sb.append(")"); + } } diff --git a/src/sqlancer/tidb/visitor/TiDBVisitor.java b/src/sqlancer/tidb/visitor/TiDBVisitor.java index 3c44229f3..847d7382f 100644 --- a/src/sqlancer/tidb/visitor/TiDBVisitor.java +++ b/src/sqlancer/tidb/visitor/TiDBVisitor.java @@ -1,16 +1,27 @@ package sqlancer.tidb.visitor; import sqlancer.tidb.ast.TiDBAggregate; +import sqlancer.tidb.ast.TiDBAlias; +import sqlancer.tidb.ast.TiDBAllOperator; +import sqlancer.tidb.ast.TiDBAnyOperator; import sqlancer.tidb.ast.TiDBCase; import sqlancer.tidb.ast.TiDBCastOperation; import sqlancer.tidb.ast.TiDBColumnReference; import sqlancer.tidb.ast.TiDBConstant; +import sqlancer.tidb.ast.TiDBExists; import sqlancer.tidb.ast.TiDBExpression; +import sqlancer.tidb.ast.TiDBExpressionBag; import sqlancer.tidb.ast.TiDBFunctionCall; +import sqlancer.tidb.ast.TiDBInOperator; import sqlancer.tidb.ast.TiDBJoin; +import sqlancer.tidb.ast.TiDBResultMap; import sqlancer.tidb.ast.TiDBSelect; +import sqlancer.tidb.ast.TiDBTableAndColumnReference; import sqlancer.tidb.ast.TiDBTableReference; import sqlancer.tidb.ast.TiDBText; +import sqlancer.tidb.ast.TiDBValues; +import sqlancer.tidb.ast.TiDBValuesRow; +import sqlancer.tidb.ast.TiDBWithClasure; public interface TiDBVisitor { @@ -35,6 +46,28 @@ default void visit(TiDBExpression expr) { visit((TiDBCastOperation) expr); } else if (expr instanceof TiDBCase) { visit((TiDBCase) expr); + } else if (expr instanceof TiDBAlias) { + visit((TiDBAlias) expr); + } else if (expr instanceof TiDBExists) { + visit((TiDBExists) expr); + } else if (expr instanceof TiDBExpressionBag) { + visit((TiDBExpressionBag) expr); + } else if (expr instanceof TiDBInOperator) { + visit((TiDBInOperator) expr); + } else if (expr instanceof TiDBTableAndColumnReference) { + visit((TiDBTableAndColumnReference) expr); + } else if (expr instanceof TiDBValues) { + visit((TiDBValues) expr); + } else if (expr instanceof TiDBValuesRow) { + visit((TiDBValuesRow) expr); + } else if (expr instanceof TiDBWithClasure) { + visit((TiDBWithClasure) expr); + } else if (expr instanceof TiDBResultMap) { + visit((TiDBResultMap) expr); + } else if (expr instanceof TiDBAllOperator) { + visit((TiDBAllOperator) expr); + } else if (expr instanceof TiDBAnyOperator) { + visit((TiDBAnyOperator) expr); } else { throw new AssertionError(expr.getClass()); } @@ -60,6 +93,19 @@ default void visit(TiDBExpression expr) { void visit(TiDBText text); + // CODDTest + void visit(TiDBAlias alias); + void visit(TiDBExists exists); + void visit(TiDBExpressionBag exprBag); + void visit(TiDBInOperator inOperation); + void visit(TiDBTableAndColumnReference tAndCRef); + void visit(TiDBValues values); + void visit(TiDBValuesRow values); + void visit(TiDBWithClasure withClasure); + void visit(TiDBResultMap tableSummary); + void visit(TiDBAllOperator allOperation); + void visit(TiDBAnyOperator anyOperation); + static String asString(TiDBExpression expr) { TiDBToStringVisitor v = new TiDBToStringVisitor(); v.visit(expr); From e9f1418c12c185188fdca992437bca343fa2b080 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Tue, 4 Mar 2025 15:45:59 +0800 Subject: [PATCH 11/16] support mysql --- src/sqlancer/mysql/MySQLErrors.java | 3 + .../mysql/MySQLExpectedValueVisitor.java | 76 ++ src/sqlancer/mysql/MySQLOptions.java | 3 + src/sqlancer/mysql/MySQLOracleFactory.java | 7 + src/sqlancer/mysql/MySQLSchema.java | 63 +- src/sqlancer/mysql/MySQLToStringVisitor.java | 240 +++- src/sqlancer/mysql/MySQLVisitor.java | 48 + src/sqlancer/mysql/ast/MySQLAggregate.java | 38 + src/sqlancer/mysql/ast/MySQLAlias.java | 24 + src/sqlancer/mysql/ast/MySQLAllOperator.java | 26 + src/sqlancer/mysql/ast/MySQLAnyOperator.java | 26 + src/sqlancer/mysql/ast/MySQLConstant.java | 4 + src/sqlancer/mysql/ast/MySQLExists.java | 14 + .../mysql/ast/MySQLExpressionBag.java | 22 + src/sqlancer/mysql/ast/MySQLResultMap.java | 41 + src/sqlancer/mysql/ast/MySQLSelect.java | 2 +- .../ast/MySQLTableAndColumnReference.java | 20 + src/sqlancer/mysql/ast/MySQLValues.java | 19 + src/sqlancer/mysql/ast/MySQLValuesRow.java | 18 + src/sqlancer/mysql/ast/MySQLWithClause.java | 29 + src/sqlancer/mysql/gen/MySQLSetGenerator.java | 5 +- .../mysql/gen/MySQLTableGenerator.java | 3 + .../mysql/oracle/MySQLCODDTestOracle.java | 1036 +++++++++++++++++ 23 files changed, 1761 insertions(+), 6 deletions(-) create mode 100644 src/sqlancer/mysql/ast/MySQLAggregate.java create mode 100644 src/sqlancer/mysql/ast/MySQLAlias.java create mode 100644 src/sqlancer/mysql/ast/MySQLAllOperator.java create mode 100644 src/sqlancer/mysql/ast/MySQLAnyOperator.java create mode 100644 src/sqlancer/mysql/ast/MySQLExpressionBag.java create mode 100644 src/sqlancer/mysql/ast/MySQLResultMap.java create mode 100644 src/sqlancer/mysql/ast/MySQLTableAndColumnReference.java create mode 100644 src/sqlancer/mysql/ast/MySQLValues.java create mode 100644 src/sqlancer/mysql/ast/MySQLValuesRow.java create mode 100644 src/sqlancer/mysql/ast/MySQLWithClause.java create mode 100644 src/sqlancer/mysql/oracle/MySQLCODDTestOracle.java diff --git a/src/sqlancer/mysql/MySQLErrors.java b/src/sqlancer/mysql/MySQLErrors.java index 13159e49e..7743f04e2 100644 --- a/src/sqlancer/mysql/MySQLErrors.java +++ b/src/sqlancer/mysql/MySQLErrors.java @@ -54,6 +54,9 @@ public static List getInsertUpdateErrors() { errors.add("cannot be null"); errors.add("Incorrect decimal value"); + errors.add("Incorrect FLOAT value"); + errors.add("Incorrect DOUBLE value"); + return errors; } diff --git a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java index 0ff8e389d..89e0920ca 100644 --- a/src/sqlancer/mysql/MySQLExpectedValueVisitor.java +++ b/src/sqlancer/mysql/MySQLExpectedValueVisitor.java @@ -1,6 +1,11 @@ package sqlancer.mysql; import sqlancer.IgnoreMeException; +import sqlancer.mysql.MySQLSchema.MySQLCompositeDataType; +import sqlancer.mysql.ast.MySQLAggregate; +import sqlancer.mysql.ast.MySQLAlias; +import sqlancer.mysql.ast.MySQLAllOperator; +import sqlancer.mysql.ast.MySQLAnyOperator; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; @@ -12,14 +17,20 @@ import sqlancer.mysql.ast.MySQLConstant; import sqlancer.mysql.ast.MySQLExists; import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLExpressionBag; import sqlancer.mysql.ast.MySQLInOperation; import sqlancer.mysql.ast.MySQLJoin; import sqlancer.mysql.ast.MySQLOrderByTerm; +import sqlancer.mysql.ast.MySQLResultMap; import sqlancer.mysql.ast.MySQLSelect; import sqlancer.mysql.ast.MySQLStringExpression; +import sqlancer.mysql.ast.MySQLTableAndColumnReference; import sqlancer.mysql.ast.MySQLTableReference; import sqlancer.mysql.ast.MySQLText; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; +import sqlancer.mysql.ast.MySQLValues; +import sqlancer.mysql.ast.MySQLValuesRow; +import sqlancer.mysql.ast.MySQLWithClause; public class MySQLExpectedValueVisitor implements MySQLVisitor { @@ -166,4 +177,69 @@ public void visit(MySQLText text) { print(text); } + @Override + public void visit(MySQLExpressionBag bag) { + visit(bag.getInnerExpr()); + } + + @Override + public void visit(MySQLValues values) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'visit'"); + } + + @Override + public void visit(MySQLTableAndColumnReference tcreference) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'visit'"); + } + + @Override + public void visit(MySQLWithClause with) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'visit'"); + } + + @Override + public void visit(MySQLValuesRow vtable) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'visit'"); + } + + @Override + public void visit(MySQLAlias alias) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'visit'"); + } + + @Override + public void visit(MySQLAggregate aggr) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'visit'"); + } + + @Override + public void visit(MySQLResultMap tSummary) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'visit'"); + } + + @Override + public void visit(MySQLAllOperator expr) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'visit'"); + } + + @Override + public void visit(MySQLAnyOperator expr) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'visit'"); + } + + @Override + public void visit(MySQLCompositeDataType type) { + // TODO Auto-generated method stub + throw new UnsupportedOperationException("Unimplemented method 'visit'"); + } + } diff --git a/src/sqlancer/mysql/MySQLOptions.java b/src/sqlancer/mysql/MySQLOptions.java index 9219073d5..b65b9c439 100644 --- a/src/sqlancer/mysql/MySQLOptions.java +++ b/src/sqlancer/mysql/MySQLOptions.java @@ -17,6 +17,9 @@ public class MySQLOptions implements DBMSSpecificOptions { @Parameter(names = "--oracle") public List oracles = Arrays.asList(MySQLOracleFactory.TLP_WHERE); + @Parameter(names = { "--coddtest-model" }, description = "Apply CODDTest on expression, subquery, or random") + public String coddTestModel = "random"; + @Override public List getTestOracleFactory() { return oracles; diff --git a/src/sqlancer/mysql/MySQLOracleFactory.java b/src/sqlancer/mysql/MySQLOracleFactory.java index a0170d685..5463b1126 100644 --- a/src/sqlancer/mysql/MySQLOracleFactory.java +++ b/src/sqlancer/mysql/MySQLOracleFactory.java @@ -13,6 +13,7 @@ import sqlancer.mysql.oracle.MySQLDQPOracle; import sqlancer.mysql.oracle.MySQLFuzzer; import sqlancer.mysql.oracle.MySQLPivotedQuerySynthesisOracle; +import sqlancer.mysql.oracle.MySQLCODDTestOracle; public enum MySQLOracleFactory implements OracleFactory { @@ -75,5 +76,11 @@ public TestOracle create(MySQLGlobalState globalState) throws public TestOracle create(MySQLGlobalState globalState) throws SQLException { return new MySQLDQPOracle(globalState); } + }, + CODDTest { + @Override + public TestOracle create(MySQLGlobalState globalState) throws SQLException { + return new MySQLCODDTestOracle(globalState); + } }; } diff --git a/src/sqlancer/mysql/MySQLSchema.java b/src/sqlancer/mysql/MySQLSchema.java index 0384f34df..ab4674f7c 100644 --- a/src/sqlancer/mysql/MySQLSchema.java +++ b/src/sqlancer/mysql/MySQLSchema.java @@ -27,7 +27,7 @@ public class MySQLSchema extends AbstractSchema { private static final int NR_SCHEMA_READ_TRIES = 10; public enum MySQLDataType { - INT, VARCHAR, FLOAT, DOUBLE, DECIMAL; + INT, VARCHAR, FLOAT, DOUBLE, DECIMAL, BOOL; public static MySQLDataType getRandom(MySQLGlobalState globalState) { if (globalState.usesPQS()) { @@ -45,6 +45,7 @@ public boolean isNumeric() { case DECIMAL: return true; case VARCHAR: + case BOOL: return false; default: throw new AssertionError(this); @@ -53,6 +54,66 @@ public boolean isNumeric() { } + public static class MySQLCompositeDataType { + + private final MySQLDataType dataType; + + // This variable is used to indicate the return value type of the query. + // Sometimes this type is hard to be mapped to a primitive type. + private String typeName = ""; + + public MySQLCompositeDataType(MySQLDataType dataType) { + this.dataType = dataType; + this.typeName = dataType.name(); + } + + public MySQLCompositeDataType(String typeName) { + typeName = typeName.split(" ")[0]; + // char, varchar, binary, varbinary, blob, tinyblob, mediumblob, longblob + // text, tinytext, mediumtext, longtext + if (typeName.toLowerCase().contains("char") || + typeName.toLowerCase().contains("binary") || + typeName.toLowerCase().contains("blob") || + typeName.toLowerCase().contains("text")) { + dataType = MySQLDataType.VARCHAR; + if (typeName.toLowerCase().contains("binary")) { + typeName = "BINARY"; + } else { + typeName = "CHAR"; + } + } + // bit, int, integer, tinyint, smallint, mediumint, bigint + // decimal + else if (typeName.toLowerCase().contains("int") || + typeName.toLowerCase().contains("bit") || + typeName.toLowerCase().contains("decimal")) { + dataType = MySQLDataType.INT; + if (typeName.toLowerCase().contains("int")) { + typeName = "SIGNED"; + } + } else if (typeName.toLowerCase().contains("float")) { + dataType = MySQLDataType.FLOAT; + } else if (typeName.toLowerCase().contains("double")) { + dataType = MySQLDataType.DOUBLE; + } else if (typeName.toLowerCase().contains("bool")) { + dataType = MySQLDataType.BOOL; + } else { + throw new AssertionError(typeName); + } + this.typeName = typeName; + } + + + public String toString() { + return this.typeName; + } + + public MySQLDataType getPrimitiveDataType() { + return dataType; + } + + } + public static class MySQLColumn extends AbstractTableColumn { private final boolean isPrimaryKey; diff --git a/src/sqlancer/mysql/MySQLToStringVisitor.java b/src/sqlancer/mysql/MySQLToStringVisitor.java index c4bce3559..fd8667208 100644 --- a/src/sqlancer/mysql/MySQLToStringVisitor.java +++ b/src/sqlancer/mysql/MySQLToStringVisitor.java @@ -1,10 +1,20 @@ package sqlancer.mysql; +import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.stream.Collectors; import sqlancer.Randomly; import sqlancer.common.visitor.ToStringVisitor; +import sqlancer.mysql.MySQLSchema.MySQLColumn; +import sqlancer.mysql.MySQLSchema.MySQLCompositeDataType; +import sqlancer.mysql.MySQLSchema.MySQLDataType; +import sqlancer.mysql.MySQLSchema.MySQLTable; +import sqlancer.mysql.ast.MySQLAggregate; +import sqlancer.mysql.ast.MySQLAlias; +import sqlancer.mysql.ast.MySQLAllOperator; +import sqlancer.mysql.ast.MySQLAnyOperator; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; @@ -14,17 +24,24 @@ import sqlancer.mysql.ast.MySQLColumnReference; import sqlancer.mysql.ast.MySQLComputableFunction; import sqlancer.mysql.ast.MySQLConstant; +import sqlancer.mysql.ast.MySQLConstant.MySQLNullConstant; import sqlancer.mysql.ast.MySQLExists; import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLExpressionBag; import sqlancer.mysql.ast.MySQLInOperation; import sqlancer.mysql.ast.MySQLJoin; import sqlancer.mysql.ast.MySQLOrderByTerm; +import sqlancer.mysql.ast.MySQLResultMap; import sqlancer.mysql.ast.MySQLOrderByTerm.MySQLOrder; import sqlancer.mysql.ast.MySQLSelect; import sqlancer.mysql.ast.MySQLStringExpression; +import sqlancer.mysql.ast.MySQLTableAndColumnReference; import sqlancer.mysql.ast.MySQLTableReference; import sqlancer.mysql.ast.MySQLText; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; +import sqlancer.mysql.ast.MySQLValues; +import sqlancer.mysql.ast.MySQLValuesRow; +import sqlancer.mysql.ast.MySQLWithClause; public class MySQLToStringVisitor extends ToStringVisitor implements MySQLVisitor { @@ -37,6 +54,10 @@ public void visitSpecific(MySQLExpression expr) { @Override public void visit(MySQLSelect s) { + if (s.getWithClause() != null) { + visit(s.getWithClause()); + sb.append(" "); + } sb.append("SELECT "); if (s.getHint() != null) { sb.append("/*+ "); @@ -69,9 +90,11 @@ public void visit(MySQLSelect s) { } visit(s.getFetchColumns().get(i)); // MySQL does not allow duplicate column names - sb.append(" AS "); - sb.append("ref"); - sb.append(ref++); + if (!(s.getFetchColumns().get(i) instanceof MySQLAlias)) { + sb.append(" AS "); + sb.append("ref"); + sb.append(ref++); + } } } sb.append(" FROM "); @@ -250,6 +273,9 @@ public void visit(MySQLOrderByTerm op) { @Override public void visit(MySQLExists op) { + if(op.isNegated()) { + sb.append(" NOT"); + } sb.append(" EXISTS ("); visit(op.getExpr()); sb.append(")"); @@ -322,4 +348,212 @@ public void visit(MySQLJoin join) { public void visit(MySQLText text) { sb.append(text.getText()); } + + @Override + public void visit(MySQLExpressionBag bag) { + visit(bag.getInnerExpr()); + } + + @Override + public void visit(MySQLTableAndColumnReference tAndCRef) { + MySQLTable table = tAndCRef.getTable(); + sb.append(table.getName()); + sb.append("("); + sb.append(table.getColumnsAsString()); + sb.append(") "); + } + + @Override + public void visit(MySQLValues values) { + LinkedHashMap> vs = values.getValues(); + int size = vs.get(vs.keySet().iterator().next()).size(); + // sb.append("VALUES "); + // sb.append("("); + for (int i = 0; i < size; i++) { + sb.append("("); + for (MySQLColumn name : vs.keySet()) { + visit(vs.get(name).get(i)); + sb.append(", "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append("), "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + } + + @Override + public void visit(MySQLValuesRow values) { + // https://database.guide/values-statement-in-mysql/ + LinkedHashMap> vs = values.getValues(); + int size = vs.get(vs.keySet().iterator().next()).size(); + sb.append("(VALUES "); + for (int i = 0; i < size; i++) { + sb.append("ROW("); + for (MySQLColumn name : vs.keySet()) { + visit(vs.get(name).get(i)); + sb.append(", "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append("), "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append(")"); + } + + @Override + public void visit(MySQLWithClause withClasure) { + sb.append("WITH "); + visit(withClasure.getLeft()); + sb.append(" AS ("); + visit(withClasure.getRight()); + sb.append(") "); + } + + @Override + public void visit(MySQLResultMap tableSummary) { + // we use CASE WHEN THEN END here + LinkedHashMap> vs = tableSummary.getDbStates(); + List results = tableSummary.getResult(); + HashMap columnType = tableSummary.getColumnType(); + + int size = vs.get(vs.keySet().iterator().next()).size(); + if (size == 0) { + sb.append("("); + for (MySQLColumnReference tr: vs.keySet()) { + visit(tr); + sb.append(" IS NULL AND "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append(")"); + return; + } + + sb.append(" CASE "); + for (int i = 0; i < size; i++) { + sb.append("WHEN "); + for (MySQLColumnReference tr: vs.keySet()) { + visit(tr); + if (vs.get(tr).get(i) instanceof MySQLNullConstant) { + sb.append(" IS NULL"); + } else { + sb.append(" = "); + if (columnType != null) { + sb.append("CONVERT("); + } + sb.append(vs.get(tr).get(i).toString()); + if (columnType != null) { + sb.append(", " + columnType.get(tr).toString().replaceAll("'", "").replaceAll("\"", "") + ")"); + } + } + sb.append(" AND "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append("THEN "); + sb.append("CONVERT("); + visit(results.get(i)); + sb.append(", " + tableSummary.getResultType().toString().replaceAll("'", "").replaceAll("\"", "") + ")"); + sb.append(" "); + } + sb.append("END "); + } + + @Override + public void visit(MySQLAllOperator allOperation) { + sb.append("("); + visit(allOperation.getLeftExpr()); + sb.append(") "); + sb.append(allOperation.getOperator()); + sb.append(" ALL ("); + if (allOperation.getRightExpr() instanceof MySQLValues) { + MySQLValues values = (MySQLValues) allOperation.getRightExpr(); + LinkedHashMap> vs = values.getValues(); + int size = vs.get(vs.keySet().iterator().next()).size(); + for (int i = 0; i < size; i++) { + if (i != 0) { + sb.append(" UNION SELECT "); + } else { + sb.append(" SELECT "); + } + + for (MySQLColumn name : vs.keySet()) { + visit(vs.get(name).get(i)); + sb.append(", "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + } + } else { + visit(allOperation.getRightExpr()); + } + sb.append(")"); + } + + @Override + public void visit(MySQLAnyOperator anyOperation) { + sb.append("("); + visit(anyOperation.getLeftExpr()); + sb.append(") "); + sb.append(anyOperation.getOperator()); + sb.append(" ANY ("); + if (anyOperation.getRightExpr() instanceof MySQLValues) { + MySQLValues values = (MySQLValues) anyOperation.getRightExpr(); + LinkedHashMap> vs = values.getValues(); + int size = vs.get(vs.keySet().iterator().next()).size(); + for (int i = 0; i < size; i++) { + if (i != 0) { + sb.append(" UNION SELECT "); + } else { + sb.append(" SELECT "); + } + + for (MySQLColumn name : vs.keySet()) { + visit(vs.get(name).get(i)); + sb.append(", "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + } + } else { + visit(anyOperation.getRightExpr()); + } + sb.append(")"); + } + + @Override + public void visit(MySQLAlias alias) { + MySQLExpression e = alias.getExpression(); + if (e instanceof MySQLSelect) { + sb.append("("); + } + visit(e); + if (e instanceof MySQLSelect) { + sb.append(")"); + } + sb.append(" AS "); + sb.append(alias.getAlias()); + } + + @Override + public void visit(MySQLAggregate aggr) { + sb.append(aggr.getFunction()); + sb.append("("); + visit(aggr.getArgs()); + sb.append(")"); + } + + @Override + public void visit(MySQLCompositeDataType type) { + sb.append(type.toString()); + } } diff --git a/src/sqlancer/mysql/MySQLVisitor.java b/src/sqlancer/mysql/MySQLVisitor.java index 7b8944f28..2ffbcff0d 100644 --- a/src/sqlancer/mysql/MySQLVisitor.java +++ b/src/sqlancer/mysql/MySQLVisitor.java @@ -1,5 +1,10 @@ package sqlancer.mysql; +import sqlancer.mysql.MySQLSchema.MySQLCompositeDataType; +import sqlancer.mysql.ast.MySQLAggregate; +import sqlancer.mysql.ast.MySQLAlias; +import sqlancer.mysql.ast.MySQLAllOperator; +import sqlancer.mysql.ast.MySQLAnyOperator; import sqlancer.mysql.ast.MySQLBetweenOperation; import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; @@ -11,14 +16,20 @@ import sqlancer.mysql.ast.MySQLConstant; import sqlancer.mysql.ast.MySQLExists; import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLExpressionBag; import sqlancer.mysql.ast.MySQLInOperation; import sqlancer.mysql.ast.MySQLJoin; import sqlancer.mysql.ast.MySQLOrderByTerm; +import sqlancer.mysql.ast.MySQLResultMap; import sqlancer.mysql.ast.MySQLSelect; import sqlancer.mysql.ast.MySQLStringExpression; +import sqlancer.mysql.ast.MySQLTableAndColumnReference; import sqlancer.mysql.ast.MySQLTableReference; import sqlancer.mysql.ast.MySQLText; import sqlancer.mysql.ast.MySQLUnaryPostfixOperation; +import sqlancer.mysql.ast.MySQLValues; +import sqlancer.mysql.ast.MySQLValuesRow; +import sqlancer.mysql.ast.MySQLWithClause; public interface MySQLVisitor { @@ -58,6 +69,21 @@ public interface MySQLVisitor { void visit(MySQLText text); + void visit(MySQLCompositeDataType type); + + // CODDTest + void visit(MySQLExpressionBag bag); + void visit(MySQLValues values); + void visit(MySQLTableAndColumnReference tcreference); + void visit(MySQLWithClause with); + void visit(MySQLValuesRow vtable); + void visit(MySQLAlias alias); + void visit(MySQLAggregate aggr); + void visit(MySQLResultMap tSummary); + void visit(MySQLAllOperator expr); + void visit(MySQLAnyOperator expr); + + default void visit(MySQLExpression expr) { if (expr instanceof MySQLConstant) { visit((MySQLConstant) expr); @@ -95,6 +121,28 @@ default void visit(MySQLExpression expr) { visit((MySQLCollate) expr); } else if (expr instanceof MySQLText) { visit((MySQLText) expr); + } else if (expr instanceof MySQLExpressionBag) { + visit((MySQLExpressionBag) expr); + } else if (expr instanceof MySQLValues) { + visit((MySQLValues) expr); + } else if (expr instanceof MySQLTableAndColumnReference) { + visit((MySQLTableAndColumnReference) expr); + } else if (expr instanceof MySQLWithClause) { + visit((MySQLWithClause) expr); + } else if (expr instanceof MySQLValuesRow) { + visit((MySQLValuesRow) expr); + } else if (expr instanceof MySQLAlias) { + visit((MySQLAlias) expr); + } else if (expr instanceof MySQLAggregate) { + visit((MySQLAggregate) expr); + } else if (expr instanceof MySQLResultMap) { + visit((MySQLResultMap) expr); + } else if (expr instanceof MySQLAllOperator) { + visit((MySQLAllOperator) expr); + } else if (expr instanceof MySQLAnyOperator) { + visit((MySQLAnyOperator) expr); + } else if (expr instanceof MySQLCompositeDataType) { + visit((MySQLCompositeDataType) expr); } else { throw new AssertionError(expr); } diff --git a/src/sqlancer/mysql/ast/MySQLAggregate.java b/src/sqlancer/mysql/ast/MySQLAggregate.java new file mode 100644 index 000000000..f8e452b29 --- /dev/null +++ b/src/sqlancer/mysql/ast/MySQLAggregate.java @@ -0,0 +1,38 @@ +package sqlancer.mysql.ast; + +import java.util.List; + +import sqlancer.Randomly; +import sqlancer.common.ast.FunctionNode; +import sqlancer.mysql.ast.MySQLAggregate.MySQLAggregateFunction; + +public class MySQLAggregate extends FunctionNode implements MySQLExpression { + + public enum MySQLAggregateFunction { + AVG(1), BIT_AND(1), BIT_OR(1), COUNT(1), SUM(1), MIN(1), MAX(1); + + private int nrArgs; + + MySQLAggregateFunction(int nrArgs) { + this.nrArgs = nrArgs; + } + + public static MySQLAggregateFunction getRandom() { + return Randomly.fromOptions(values()); + } + + public int getNrArgs() { + return nrArgs; + } + + } + + public MySQLAggregate(List args, MySQLAggregateFunction func) { + super(func, args); + } + + @Override + public MySQLConstant getExpectedValue() { + return MySQLConstant.createNullConstant(); + } +} diff --git a/src/sqlancer/mysql/ast/MySQLAlias.java b/src/sqlancer/mysql/ast/MySQLAlias.java new file mode 100644 index 000000000..f76b43f7d --- /dev/null +++ b/src/sqlancer/mysql/ast/MySQLAlias.java @@ -0,0 +1,24 @@ +package sqlancer.mysql.ast; + +public class MySQLAlias implements MySQLExpression { + private final MySQLExpression expr; + private final String alias; + + public MySQLAlias(MySQLExpression expr, String alias) { + this.expr = expr; + this.alias = alias; + } + + public MySQLExpression getExpression() { + return expr; + } + + public String getAlias() { + return alias; + } + + @Override + public MySQLConstant getExpectedValue() { + return MySQLConstant.createNullConstant(); + } +} diff --git a/src/sqlancer/mysql/ast/MySQLAllOperator.java b/src/sqlancer/mysql/ast/MySQLAllOperator.java new file mode 100644 index 000000000..21fca510c --- /dev/null +++ b/src/sqlancer/mysql/ast/MySQLAllOperator.java @@ -0,0 +1,26 @@ +package sqlancer.mysql.ast; + +import sqlancer.mysql.ast.MySQLBinaryComparisonOperation.BinaryComparisonOperator; + +public class MySQLAllOperator implements MySQLExpression { + private final MySQLExpression leftExpr; + private final MySQLExpression rightExpr; + private final BinaryComparisonOperator op; + + + public MySQLAllOperator(MySQLExpression leftExpr, MySQLExpression rightExpr, BinaryComparisonOperator op) { + this.leftExpr = leftExpr; + this.rightExpr = rightExpr; + this.op = op; + } + + public MySQLExpression getLeftExpr() { + return leftExpr; + } + public MySQLExpression getRightExpr() { + return rightExpr; + } + public String getOperator() { + return op.getTextRepresentation(); + } +} diff --git a/src/sqlancer/mysql/ast/MySQLAnyOperator.java b/src/sqlancer/mysql/ast/MySQLAnyOperator.java new file mode 100644 index 000000000..7234fc1af --- /dev/null +++ b/src/sqlancer/mysql/ast/MySQLAnyOperator.java @@ -0,0 +1,26 @@ +package sqlancer.mysql.ast; + +import sqlancer.mysql.ast.MySQLBinaryComparisonOperation.BinaryComparisonOperator; + +public class MySQLAnyOperator implements MySQLExpression { + private final MySQLExpression leftExpr; + private final MySQLExpression rightExpr; + private final BinaryComparisonOperator op; + + + public MySQLAnyOperator(MySQLExpression leftExpr, MySQLExpression rightExpr, BinaryComparisonOperator op) { + this.leftExpr = leftExpr; + this.rightExpr = rightExpr; + this.op = op; + } + + public MySQLExpression getLeftExpr() { + return leftExpr; + } + public MySQLExpression getRightExpr() { + return rightExpr; + } + public String getOperator() { + return op.getTextRepresentation(); + } +} diff --git a/src/sqlancer/mysql/ast/MySQLConstant.java b/src/sqlancer/mysql/ast/MySQLConstant.java index 2e4922f8e..058febb08 100644 --- a/src/sqlancer/mysql/ast/MySQLConstant.java +++ b/src/sqlancer/mysql/ast/MySQLConstant.java @@ -413,6 +413,10 @@ public static MySQLConstant createIntConstantNotAsBoolean(long value) { return new MySQLIntConstant(value, String.valueOf(value)); } + public static MySQLConstant createDoubleConstant(double value) { + return new MySQLDoubleConstant(value); + } + @Override public MySQLConstant getExpectedValue() { return this; diff --git a/src/sqlancer/mysql/ast/MySQLExists.java b/src/sqlancer/mysql/ast/MySQLExists.java index 46a8b5436..c0c214bd5 100644 --- a/src/sqlancer/mysql/ast/MySQLExists.java +++ b/src/sqlancer/mysql/ast/MySQLExists.java @@ -4,6 +4,7 @@ public class MySQLExists implements MySQLExpression { private final MySQLExpression expr; private final MySQLConstant expected; + private boolean negated = false; public MySQLExists(MySQLExpression expr, MySQLConstant expectedValue) { this.expr = expr; @@ -18,10 +19,23 @@ public MySQLExists(MySQLExpression expr) { } } + public MySQLExists(MySQLExpression expr, boolean isNegated) { + this.expr = expr; + negated = isNegated; + this.expected = expr.getExpectedValue(); + if (expected == null) { + throw new AssertionError(); + } + } + public MySQLExpression getExpr() { return expr; } + public boolean isNegated() { + return negated; + } + @Override public MySQLConstant getExpectedValue() { return expected; diff --git a/src/sqlancer/mysql/ast/MySQLExpressionBag.java b/src/sqlancer/mysql/ast/MySQLExpressionBag.java new file mode 100644 index 000000000..e39b1e99a --- /dev/null +++ b/src/sqlancer/mysql/ast/MySQLExpressionBag.java @@ -0,0 +1,22 @@ +package sqlancer.mysql.ast; + +public class MySQLExpressionBag implements MySQLExpression { + private MySQLExpression innerExpr; + + public MySQLExpressionBag(MySQLExpression innerExpr) { + this.innerExpr = innerExpr; + } + + public void updateInnerExpr(MySQLExpression innerExpr) { + this.innerExpr = innerExpr; + } + + public MySQLExpression getInnerExpr() { + return innerExpr; + } + + @Override + public MySQLConstant getExpectedValue() { + return innerExpr.getExpectedValue(); + } +} diff --git a/src/sqlancer/mysql/ast/MySQLResultMap.java b/src/sqlancer/mysql/ast/MySQLResultMap.java new file mode 100644 index 000000000..8a85d90fb --- /dev/null +++ b/src/sqlancer/mysql/ast/MySQLResultMap.java @@ -0,0 +1,41 @@ +package sqlancer.mysql.ast; + +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; + +import sqlancer.mysql.MySQLSchema.MySQLCompositeDataType;; + +public class MySQLResultMap implements MySQLExpression { + private final LinkedHashMap> DBStates; + private final List results; + private final HashMap columnType; + MySQLCompositeDataType resultType; + + public MySQLResultMap(LinkedHashMap> s, + HashMap ct, List r, MySQLCompositeDataType rt) { + this.DBStates = s; + this.results = r; + this.resultType = rt; + this.columnType = ct; + if (s.get(s.keySet().iterator().next()).size() != r.size()) { + throw new AssertionError(); + } + } + + public LinkedHashMap> getDbStates() { + return this.DBStates; + } + + public List getResult() { + return this.results; + } + + public HashMap getColumnType() { + return this.columnType; + } + + public MySQLCompositeDataType getResultType() { + return this.resultType; + } +} diff --git a/src/sqlancer/mysql/ast/MySQLSelect.java b/src/sqlancer/mysql/ast/MySQLSelect.java index 7b9243c20..df644adb2 100644 --- a/src/sqlancer/mysql/ast/MySQLSelect.java +++ b/src/sqlancer/mysql/ast/MySQLSelect.java @@ -43,7 +43,7 @@ public List getModifiers() { @Override public MySQLConstant getExpectedValue() { - return null; + return MySQLConstant.createNullConstant(); } public void setHint(MySQLText hint) { diff --git a/src/sqlancer/mysql/ast/MySQLTableAndColumnReference.java b/src/sqlancer/mysql/ast/MySQLTableAndColumnReference.java new file mode 100644 index 000000000..fceafe5e7 --- /dev/null +++ b/src/sqlancer/mysql/ast/MySQLTableAndColumnReference.java @@ -0,0 +1,20 @@ +package sqlancer.mysql.ast; + +import sqlancer.mysql.MySQLSchema.MySQLTable; + +public class MySQLTableAndColumnReference implements MySQLExpression { + private final MySQLTable table; + + public MySQLTableAndColumnReference(MySQLTable table) { + this.table = table; + } + + public MySQLTable getTable() { + return table; + } + + @Override + public MySQLConstant getExpectedValue() { + return MySQLConstant.createNullConstant(); + } +} diff --git a/src/sqlancer/mysql/ast/MySQLValues.java b/src/sqlancer/mysql/ast/MySQLValues.java new file mode 100644 index 000000000..990cfe99a --- /dev/null +++ b/src/sqlancer/mysql/ast/MySQLValues.java @@ -0,0 +1,19 @@ +package sqlancer.mysql.ast; + +import java.util.LinkedHashMap; +import java.util.List; + +import sqlancer.mysql.MySQLSchema.MySQLColumn; + +public class MySQLValues implements MySQLExpression { + + private final LinkedHashMap> values; + + public MySQLValues(LinkedHashMap> v) { + this.values = v; + } + + public LinkedHashMap> getValues() { + return this.values; + } +} diff --git a/src/sqlancer/mysql/ast/MySQLValuesRow.java b/src/sqlancer/mysql/ast/MySQLValuesRow.java new file mode 100644 index 000000000..1bc0296a3 --- /dev/null +++ b/src/sqlancer/mysql/ast/MySQLValuesRow.java @@ -0,0 +1,18 @@ +package sqlancer.mysql.ast; + +import java.util.LinkedHashMap; +import java.util.List; + +import sqlancer.mysql.MySQLSchema.MySQLColumn; + +public class MySQLValuesRow implements MySQLExpression { + private final LinkedHashMap> values; + + public MySQLValuesRow(LinkedHashMap> values) { + this.values = values; + } + + public LinkedHashMap> getValues() { + return this.values; + } +} diff --git a/src/sqlancer/mysql/ast/MySQLWithClause.java b/src/sqlancer/mysql/ast/MySQLWithClause.java new file mode 100644 index 000000000..71f5e26de --- /dev/null +++ b/src/sqlancer/mysql/ast/MySQLWithClause.java @@ -0,0 +1,29 @@ +package sqlancer.mysql.ast; + +public class MySQLWithClause implements MySQLExpression { + private MySQLExpression left; + private MySQLExpression right; + + public MySQLWithClause(MySQLExpression left, MySQLExpression right) { + this.left = left; + this.right = right; + } + + public MySQLExpression getLeft() { + return this.left; + } + + public MySQLExpression getRight() { + return this.right; + } + + public void updateRight(MySQLExpression right) { + this.right = right; + } + + @Override + public MySQLConstant getExpectedValue() { + return MySQLConstant.createNullConstant(); + } + +} diff --git a/src/sqlancer/mysql/gen/MySQLSetGenerator.java b/src/sqlancer/mysql/gen/MySQLSetGenerator.java index 79333eb36..8f21fb85b 100644 --- a/src/sqlancer/mysql/gen/MySQLSetGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLSetGenerator.java @@ -99,7 +99,10 @@ private enum Action { SCHEMA_DEFINITION_CACHE("schema_definition_cache", (r) -> r.getLong(256, 524288), Scope.GLOBAL), // SHOW_CREATE_TABLE_VERBOSITY("show_create_table_verbosity", (r) -> Randomly.fromOptions("OFF", "ON"), Scope.GLOBAL, Scope.SESSION), // - SHOW_OLD_TEMPORALS("show_old_temporals", (r) -> Randomly.fromOptions("OFF", "ON"), Scope.GLOBAL, Scope.SESSION), + /* + * show_old_temporals was removed in MySQL 8.4 https://dev.mysql.com/doc/refman/8.4/en/added-deprecated-removed.html + */ + // SHOW_OLD_TEMPORALS("show_old_temporals", (r) -> Randomly.fromOptions("OFF", "ON"), Scope.GLOBAL, Scope.SESSION), /* * sort_buffer_size is commented out as a workaround for https://bugs.mysql.com/bug.php?id=95969 */ diff --git a/src/sqlancer/mysql/gen/MySQLTableGenerator.java b/src/sqlancer/mysql/gen/MySQLTableGenerator.java index bc0533295..94d9239ee 100644 --- a/src/sqlancer/mysql/gen/MySQLTableGenerator.java +++ b/src/sqlancer/mysql/gen/MySQLTableGenerator.java @@ -352,6 +352,9 @@ private void appendType(MySQLDataType randomType) { sb.append(Randomly.fromOptions("DOUBLE", "FLOAT")); optionallyAddPrecisionAndScale(sb); break; + case BOOL: + sb.append(Randomly.fromOptions("BOOL", "BOOLEAN")); + break; default: throw new AssertionError(); } diff --git a/src/sqlancer/mysql/oracle/MySQLCODDTestOracle.java b/src/sqlancer/mysql/oracle/MySQLCODDTestOracle.java new file mode 100644 index 000000000..5facf62c2 --- /dev/null +++ b/src/sqlancer/mysql/oracle/MySQLCODDTestOracle.java @@ -0,0 +1,1036 @@ +package sqlancer.mysql.oracle; + +import java.sql.ResultSet; +import java.sql.ResultSetMetaData; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import org.apache.arrow.driver.jdbc.shaded.org.apache.arrow.flatbuf.Bool; + +import sqlancer.IgnoreMeException; +import sqlancer.Main; +import sqlancer.Randomly; +import sqlancer.Reproducer; +import sqlancer.common.oracle.CODDTestBase; +import sqlancer.common.oracle.TestOracle; +import sqlancer.mysql.MySQLErrors; +import sqlancer.mysql.gen.MySQLExpressionGenerator; +import sqlancer.mysql.MySQLSchema; +import sqlancer.mysql.MySQLGlobalState; +import sqlancer.mysql.MySQLSchema.MySQLColumn; +import sqlancer.mysql.MySQLSchema.MySQLCompositeDataType; +import sqlancer.mysql.MySQLSchema.MySQLTable; +import sqlancer.mysql.MySQLSchema.MySQLTables; +import sqlancer.mysql.ast.MySQLAggregate; +import sqlancer.mysql.ast.MySQLAlias; +import sqlancer.mysql.ast.MySQLAllOperator; +import sqlancer.mysql.ast.MySQLAnyOperator; +import sqlancer.mysql.ast.MySQLBinaryComparisonOperation; +import sqlancer.mysql.ast.MySQLBinaryLogicalOperation; +import sqlancer.mysql.ast.MySQLColumnReference; +import sqlancer.mysql.ast.MySQLConstant; +import sqlancer.mysql.ast.MySQLExists; +import sqlancer.mysql.ast.MySQLExpression; +import sqlancer.mysql.ast.MySQLExpressionBag; +import sqlancer.mysql.ast.MySQLInOperation; +import sqlancer.mysql.ast.MySQLJoin; +import sqlancer.mysql.ast.MySQLJoin.JoinType; +import sqlancer.mysql.ast.MySQLOrderByTerm; +import sqlancer.mysql.ast.MySQLSelect; +import sqlancer.mysql.ast.MySQLTableAndColumnReference; +import sqlancer.mysql.ast.MySQLTableReference; +import sqlancer.mysql.ast.MySQLResultMap; +import sqlancer.mysql.ast.MySQLValues; +import sqlancer.mysql.ast.MySQLValuesRow; +import sqlancer.mysql.ast.MySQLWithClause; +import sqlancer.mysql.ast.MySQLAggregate.MySQLAggregateFunction; +import sqlancer.mysql.ast.MySQLBinaryComparisonOperation.BinaryComparisonOperator; +import sqlancer.mysql.ast.MySQLBinaryLogicalOperation.MySQLBinaryLogicalOperator; +import sqlancer.mysql.ast.MySQLOrderByTerm.MySQLOrder; +import sqlancer.mysql.MySQLVisitor; + +public class MySQLCODDTestOracle extends CODDTestBase implements TestOracle { + + private final MySQLSchema s; + private MySQLExpressionGenerator gen; + private Reproducer reproducer; + + private String tempTableName = "temp_table"; + + private MySQLExpression foldedExpr; + private MySQLExpression constantResOfFoldedExpr; + + + private List tablesFromOuterContext = new ArrayList<>(); + private List joinsInExpr = null; + + Map> auxiliaryQueryResult = new HashMap<>(); + Map> selectResult = new HashMap<>(); + + Boolean useSubqueryAsFoldedExpr; + Boolean useCorrelatedSubqueryAsFoldedExpr; + + MySQLCompositeDataType foldedExpressionReturnType = null; + + public MySQLCODDTestOracle(MySQLGlobalState globalState) { + super(globalState); + this.s = globalState.getSchema(); + MySQLErrors.addExpressionErrors(errors); + + // the following two errors generated as we only generate a constant in the group expression + errors.add("Unknown column"); + errors.add("Can't group on"); + + // this is triggered when generate a wrong group + errors.add("this is incompatible with sql_mode=only_full_group_by"); + } + + @Override + public void check() throws Exception { + + reproducer = null; + + joinsInExpr = null; + tablesFromOuterContext.clear(); + + useSubqueryAsFoldedExpr = useSubquery(); + useCorrelatedSubqueryAsFoldedExpr = useCorrelatedSubquery(); + + + MySQLSelect auxiliaryQuery = null; + + if (useSubqueryAsFoldedExpr) { + if (useCorrelatedSubqueryAsFoldedExpr) { + auxiliaryQuery = genSelectWithCorrelatedSubquery(); + auxiliaryQueryString = MySQLVisitor.asString(auxiliaryQuery); + auxiliaryQueryResult.putAll(selectResult); + } else { + auxiliaryQuery = genSelectExpression(null, null, null); + auxiliaryQueryString = MySQLVisitor.asString(auxiliaryQuery); + auxiliaryQueryResult = getQueryResult(auxiliaryQueryString, state); + } + } else { + auxiliaryQuery = genSimpleSelect(); + auxiliaryQueryString = MySQLVisitor.asString(auxiliaryQuery); + auxiliaryQueryResult.putAll(selectResult); + } + + + MySQLSelect originalQuery = null; + + Map> foldedResult = new HashMap<>(); + Map> originalResult = new HashMap<>(); + + // dependent expression + if (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) { + MySQLExpressionBag specificCondition = new MySQLExpressionBag(this.foldedExpr); + originalQuery = this.genSelectExpression(null, specificCondition, null); + originalQueryString = MySQLVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + specificCondition.updateInnerExpr(this.constantResOfFoldedExpr); + foldedQueryString = MySQLVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + + // independent expression + // empty result, put the inner query in (NOT) EXIST + else if (auxiliaryQueryResult.size() == 0 || auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().iterator().next()).size() == 0) { + boolean isNegated = Randomly.getBoolean() ? false : true; + + // original query + MySQLExists existExpr = new MySQLExists(auxiliaryQuery, isNegated); + MySQLExpressionBag specificCondition = new MySQLExpressionBag(existExpr); + + originalQuery = this.genSelectExpression(null, specificCondition, foldedExpressionReturnType); + originalQueryString = MySQLVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + + // folded query + MySQLExpression equivalentExpr = MySQLConstant.createBoolean(isNegated); + specificCondition.updateInnerExpr(equivalentExpr); + foldedQueryString = MySQLVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + // Scalar Subquery: 1 column and 1 row, consider the inner query as a constant + else if (auxiliaryQueryResult.size() == 1 && auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]).size() == 1 && Randomly.getBoolean()) { + // original query + MySQLExpressionBag specificCondition = new MySQLExpressionBag(auxiliaryQuery); + originalQuery = this.genSelectExpression(null, specificCondition, getColumnTypeFromSelect(auxiliaryQuery).get(0)); + originalQueryString = MySQLVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + MySQLExpression equivalentExpr = auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]).get(0); + specificCondition.updateInnerExpr(equivalentExpr);; + foldedQueryString = MySQLVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + // one column + else if (auxiliaryQueryResult.size() == 1 && Randomly.getBoolean()) { + // original query + List columns = s.getRandomTableNonEmptyTables().getColumns(); + MySQLColumnReference selectedColumn = new MySQLColumnReference(Randomly.fromList(columns), null); + MySQLTable selectedTable = selectedColumn.getColumn().getTable(); + MySQLTableReference selectedTableRef = new MySQLTableReference(selectedTable); + MySQLExpressionBag tableBag = new MySQLExpressionBag(selectedTableRef); + + MySQLInOperation optInOperation = new MySQLInOperation(selectedColumn, Arrays.asList((MySQLExpression) auxiliaryQuery), true); + MySQLExpressionBag specificCondition = new MySQLExpressionBag(optInOperation); + originalQuery = this.genSelectExpression(tableBag, specificCondition, null); + originalQueryString = MySQLVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + List vs = new ArrayList<>(); + for (MySQLConstant c: auxiliaryQueryResult.values().iterator().next()) { + vs.add(c); + } + MySQLInOperation refInOperation = new MySQLInOperation(selectedColumn, vs, true); + specificCondition.updateInnerExpr(refInOperation); + foldedQueryString = MySQLVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + + // ALL + else if (auxiliaryQueryResult.size() == 1 && Randomly.getBoolean()) { + // original query + List columns = s.getRandomTableNonEmptyTables().getColumns(); + MySQLColumnReference selectedColumn = new MySQLColumnReference(Randomly.fromList(columns), null); + MySQLTable selectedTable = selectedColumn.getColumn().getTable(); + MySQLTableReference selectedTableRef = new MySQLTableReference(selectedTable); + MySQLExpressionBag tableBag = new MySQLExpressionBag(selectedTableRef); + + MySQLExpressionGenerator exprGen = new MySQLExpressionGenerator(state).setColumns(Arrays.asList(selectedColumn.getColumn())); + MySQLExpression allOptLeft = genCondition(exprGen, null, null); + BinaryComparisonOperator allOperator = BinaryComparisonOperator.getRandom(); + while (allOperator == BinaryComparisonOperator.LIKE) { + allOperator = BinaryComparisonOperator.getRandom(); + } + MySQLAllOperator optAllOperation = new MySQLAllOperator(allOptLeft, auxiliaryQuery, allOperator); + MySQLExpressionBag specificCondition = new MySQLExpressionBag(optAllOperation); + originalQuery = this.genSelectExpression(tableBag, specificCondition, null); + originalQueryString = MySQLVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + MySQLColumn tempColumn = new MySQLColumn("c0", getColumnTypeFromSelect(auxiliaryQuery).get(0).getPrimitiveDataType(), false, 0); + LinkedHashMap> value = new LinkedHashMap<>(); + value.put(tempColumn, auxiliaryQueryResult.values().iterator().next()); + MySQLValues refValues = new MySQLValues(value); + MySQLAllOperator refAllOperation = new MySQLAllOperator(allOptLeft, refValues, allOperator); + specificCondition.updateInnerExpr(refAllOperation); + foldedQueryString = MySQLVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + + // ANY + else if (auxiliaryQueryResult.size() == 1 && Randomly.getBoolean()) { + // original query + List columns = s.getRandomTableNonEmptyTables().getColumns(); + MySQLColumnReference selectedColumn = new MySQLColumnReference(Randomly.fromList(columns), null); + MySQLTable selectedTable = selectedColumn.getColumn().getTable(); + MySQLTableReference selectedTableRef = new MySQLTableReference(selectedTable); + MySQLExpressionBag tableBag = new MySQLExpressionBag(selectedTableRef); + + MySQLExpressionGenerator exprGen = new MySQLExpressionGenerator(state).setColumns(Arrays.asList(selectedColumn.getColumn())); + MySQLExpression anyOptLeft = genCondition(exprGen, null, null); + BinaryComparisonOperator anyOperator = BinaryComparisonOperator.getRandom(); + while (anyOperator == BinaryComparisonOperator.LIKE) { + anyOperator = BinaryComparisonOperator.getRandom(); + } + MySQLAnyOperator optAnyOperation = new MySQLAnyOperator(anyOptLeft, auxiliaryQuery, anyOperator); + MySQLExpressionBag specificCondition = new MySQLExpressionBag(optAnyOperation); + originalQuery = this.genSelectExpression(tableBag, specificCondition, null); + originalQueryString = MySQLVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + MySQLColumn tempColumn = new MySQLColumn("c0", getColumnTypeFromSelect(auxiliaryQuery).get(0).getPrimitiveDataType(), false, 0); + LinkedHashMap> value = new LinkedHashMap<>(); + value.put(tempColumn, auxiliaryQueryResult.values().iterator().next()); + MySQLValues refValues = new MySQLValues(value); + MySQLAnyOperator refAnyOperation = new MySQLAnyOperator(anyOptLeft, refValues, anyOperator); + specificCondition.updateInnerExpr(refAnyOperation); + foldedQueryString = MySQLVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } + + // Row Subquery + else { + // original query + MySQLTable temporaryTable = this.genTemporaryTable(auxiliaryQuery, tempTableName); + MySQLTableReference tempTableRef = new MySQLTableReference(temporaryTable); + + LinkedHashMap> value = new LinkedHashMap<>(); + for (MySQLColumn c: temporaryTable.getColumns()) { + value.put(c, auxiliaryQueryResult.get(c.getName())); + } + MySQLValuesRow resValues = new MySQLValuesRow(value); + + MySQLExpressionBag tempTableRefBag = new MySQLExpressionBag(tempTableRef); + MySQLTableAndColumnReference tableAndColumnRef = new MySQLTableAndColumnReference(temporaryTable); + MySQLWithClause withClasure = null; + if (Randomly.getBoolean()) { + withClasure = new MySQLWithClause(tableAndColumnRef, auxiliaryQuery); + } else { + withClasure = new MySQLWithClause(tableAndColumnRef, resValues); + } + originalQuery = genSelectExpression(tempTableRefBag, null, null); + originalQuery.setWithClause(withClasure); + originalQueryString = MySQLVisitor.asString(originalQuery); + originalResult = getQueryResult(originalQueryString, state); + + // folded query + if (Randomly.getBoolean()) { + // folded query: FROM VALUES () AS table, mysql seems not support this + originalQuery.setWithClause(null); + MySQLAlias alias = new MySQLAlias(resValues, MySQLVisitor.asString(tableAndColumnRef)); + tempTableRefBag.updateInnerExpr(alias); + foldedQueryString = MySQLVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } else if (Randomly.getBoolean()) { + // folded query: SELECT FROM () AS table + originalQuery.setWithClause(null); + MySQLAlias alias = null; + if (Randomly.getBoolean()) { + alias = new MySQLAlias(auxiliaryQuery, MySQLVisitor.asString(tempTableRef)); + } else { + // SELECT * FROM (VALUES ROW(1)) AS t2(c0); is not supported in MySQL + alias = new MySQLAlias(resValues, MySQLVisitor.asString(tableAndColumnRef)); + } + tempTableRefBag.updateInnerExpr(alias); + foldedQueryString = MySQLVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } else { + // folded query: CREATE the table + try { + this.createTemporaryTable(auxiliaryQuery, tempTableName, MySQLVisitor.asString(resValues)); + originalQuery.setWithClause(null); + foldedQueryString = MySQLVisitor.asString(originalQuery); + foldedResult = getQueryResult(foldedQueryString, state); + } finally { + dropTemporaryTable(tempTableName); + } + } + } + + if (foldedResult == null || originalResult == null) { + throw new IgnoreMeException(); + } + + if (!compareResult(foldedResult, originalResult)) { + reproducer = null; // TODO + state.getState().getLocalState().log(auxiliaryQueryString + ";\n" +foldedQueryString + ";\n" + originalQueryString + ";"); + throw new AssertionError(auxiliaryQueryResult.toString() + " " +foldedResult.toString() + " " + originalResult.toString()); + } + } + + private MySQLSelect genSelectExpression(MySQLExpressionBag tableBag, MySQLExpression specificCondition, MySQLCompositeDataType conditionType) { + MySQLTables randomTables = s.getRandomTableNonEmptyTables(); + if (!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) { + for (MySQLTable t : this.tablesFromOuterContext) { + randomTables.addTable(t); + } + if (this.joinsInExpr != null) { + for (MySQLJoin j : this.joinsInExpr) { + randomTables.removeTable(j.getTable()); + } + } + } + MySQLTable tempTable = null; + MySQLTableReference tableRef = null; + if (tableBag != null) { + tableRef = (MySQLTableReference) tableBag.getInnerExpr(); + tempTable = tableRef.getTable(); + } + List columns = randomTables.getColumns(); + if (tempTable != null) { + columns.addAll(tempTable.getColumns()); + } + if ((!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr)) && this.joinsInExpr != null) { + for (MySQLJoin j : this.joinsInExpr) { + MySQLTable t = j.getTable(); + columns.addAll(t.getColumns()); + } + } + gen = new MySQLExpressionGenerator(state).setColumns(columns); + List tables = randomTables.getTables(); + List tableRefs = tables.stream().map(t -> new MySQLTableReference(t)).collect(Collectors.toList()); + + MySQLSelect select = new MySQLSelect(); + + // MySQL currently not support subquery in ON + // List joins = genJoinExpressions(tableRefs, state, specificCondition, conditionType); + List joins = new ArrayList<>(); + if ((!useSubqueryAsFoldedExpr || (useSubqueryAsFoldedExpr && useCorrelatedSubqueryAsFoldedExpr))) { + if (this.joinsInExpr != null) { + joins.addAll(this.joinsInExpr); + this.joinsInExpr = null; + } + } + else if (Randomly.getBoolean()) { + joins = genJoinExpressions(tableRefs, state, specificCondition, conditionType); + } + if (joins.size() > 0) { + select.setJoinClauses(joins); + } + + if (tableBag != null) { + MySQLTableReference outerTable = (MySQLTableReference) tableBag.getInnerExpr(); + boolean isContained = false; + for (MySQLExpression e: tableRefs) { + MySQLTableReference tr = (MySQLTableReference) e; + if (tr.getTable().getName().equals(outerTable.getTable().getName())) { + isContained = true; + } + } + if (joins.size() > 0) { + for (MySQLJoin j : joins) { + MySQLTable t = j.getTable(); + if (t.getName().equals(outerTable.getTable().getName())) { + isContained = true; + } + } + } + if (!isContained) { + tableRefs.add(tableBag); + } + } + + select.setFromList(tableRefs); + + select.setWhereClause(genCondition(gen, specificCondition, conditionType)); + + if (Randomly.getBooleanWithSmallProbability()) { + select.setOrderByClauses(genOrderBys(gen, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null, conditionType)); + } + + if (Randomly.getBoolean()) { + List selectedColumns = Randomly.nonEmptySubset(columns); + List selectedAlias = new LinkedList<>(); + for (int i = 0; i < selectedColumns.size(); ++i) { + MySQLColumnReference originalName = new MySQLColumnReference(selectedColumns.get(i), null); + MySQLAlias columnAlias = new MySQLAlias(originalName, "c" + String.valueOf(i)); + selectedAlias.add(columnAlias); + } + select.setFetchColumns(selectedAlias); + } else { + MySQLColumn selectedColumn = Randomly.fromList(columns); + MySQLColumnReference aggr = new MySQLColumnReference(selectedColumn, null); + MySQLAggregateFunction windowFunction = MySQLAggregateFunction.getRandom(); + // one bug reported about this + while(windowFunction == MySQLAggregateFunction.BIT_AND || windowFunction == MySQLAggregateFunction.BIT_OR) { + windowFunction = MySQLAggregateFunction.getRandom(); + } + MySQLExpression originalName = new MySQLAggregate(Arrays.asList(aggr), windowFunction); + MySQLAlias columnAlias = new MySQLAlias(originalName, "c0"); + select.setFetchColumns(Arrays.asList(columnAlias)); + // there are many syntax error in group by with subquery, just remove it + select.setGroupByExpressions(genGroupBys(Randomly.nonEmptySubset(columns), Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null, conditionType)); + select.setGroupByExpressions(gen.generateGroupBys()); + + // gen having + // there is an error in having has not been fixed: unknown column in having clause + if (Randomly.getBooleanWithRatherLowProbability()) { + MySQLExpressionGenerator havingGen = new MySQLExpressionGenerator(state).setColumns(columns); + select.setHavingClause(genCondition(havingGen, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null, conditionType)); + } + } + return select; + } + + private MySQLExpression genCondition(MySQLExpressionGenerator conGen, MySQLExpression specificCondition, MySQLCompositeDataType conditionType) { + MySQLExpression randomCondition = conGen.generateBooleanExpression(); + if (specificCondition != null) { + if (conditionType == null) { + randomCondition = new MySQLBinaryLogicalOperation(randomCondition, specificCondition, MySQLBinaryLogicalOperator.getRandom()); + } else { + switch(conditionType.getPrimitiveDataType()) { + // case BOOL: + // randomCondition = new MySQLBinaryLogicalOperation(randomCondition, specificCondition, MySQLBinaryLogicalOperator.getRandom()); + // break; + + case DECIMAL: + case INT: + case VARCHAR: + case FLOAT: + case DOUBLE: + randomCondition = new MySQLBinaryComparisonOperation(randomCondition, specificCondition, BinaryComparisonOperator.getRandom()); + break; + default: + randomCondition = new MySQLBinaryLogicalOperation(randomCondition, specificCondition, MySQLBinaryLogicalOperator.getRandom()); + break; + } + } + } + return randomCondition; + } + + private List genJoinExpressions(List tableList, MySQLGlobalState globalState, MySQLExpression specificCondition, MySQLCompositeDataType conditionType) { + List joinExpressions = new ArrayList<>(); + while (tableList.size() >= 2 && Randomly.getBoolean()) { + MySQLTableReference rightTable = (MySQLTableReference) tableList.remove(0); + List columns = new ArrayList<>(rightTable.getTable().getColumns()); + + MySQLExpressionGenerator joinGen = new MySQLExpressionGenerator(globalState).setColumns(columns); + MySQLExpression randomCondition = genCondition(joinGen, specificCondition, conditionType); + JoinType selectedOption = Randomly.fromList(Arrays.asList(JoinType.values())); + joinExpressions.add(new MySQLJoin(rightTable.getTable(), randomCondition, selectedOption)); + } + return joinExpressions; + } + + public List genOrderBys(MySQLExpressionGenerator orderByGen, MySQLExpression specificCondition, MySQLCompositeDataType conditionType) { + int exprNum = Randomly.smallNumber() + 1; + List newExpressions = new ArrayList<>(); + for (int i = 0; i < exprNum; ++i) { + MySQLExpression condition = genCondition(orderByGen, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null, conditionType); + if (Randomly.getBoolean()) { + condition = new MySQLOrderByTerm(condition, MySQLOrder.getRandomOrder()); + } + newExpressions.add(condition); + } + return newExpressions; + } + + private List genGroupBys(List columns, MySQLExpression specificCondition, MySQLCompositeDataType conditionType) { + MySQLExpressionGenerator groupByGen = new MySQLExpressionGenerator(state).setColumns(columns); + int exprNum = Randomly.smallNumber() + 1; + List newExpressions = new ArrayList<>(); + for (int i = 0; i < exprNum; ++i) { + MySQLExpression condition = genCondition(groupByGen, Randomly.getBooleanWithRatherLowProbability() ? specificCondition : null, conditionType); + newExpressions.add(condition); + } + return newExpressions; + } + + private Map> getQueryResult(String queryString, MySQLGlobalState state) throws SQLException { + Map> result = new LinkedHashMap<>(); + if (options.logEachSelect()) { + logger.writeCurrent(queryString); + } + Statement stmt = null; + try { + stmt = this.con.createStatement(); + ResultSet rs = null; + try { + rs = stmt.executeQuery(queryString); + ResultSetMetaData metaData = rs.getMetaData(); + Integer columnCount = metaData.getColumnCount(); + Map idxNameMap = new HashMap<>(); + for (int i = 1; i <= columnCount; i++) { + result.put("c" + String.valueOf(i-1), new ArrayList<>()); + idxNameMap.put(i, "c" + String.valueOf(i-1)); + } + + int resultRows = 0; + while (rs.next()) { + for (int i = 1; i <= columnCount; i++) { + try { + Object value = rs.getObject(i); + MySQLConstant constant; + if (rs.wasNull()) { + constant = MySQLConstant.createNullConstant(); + } + + else if (value instanceof Integer) { + constant = MySQLConstant.createIntConstant(Long.valueOf((Integer) value)); + } else if (value instanceof Short) { + constant = MySQLConstant.createIntConstant(Long.valueOf((Short) value)); + } else if (value instanceof Long) { + constant = MySQLConstant.createIntConstant((Long) value); + } else if (value instanceof java.math.BigInteger) { + constant = MySQLConstant.createIntConstant(((java.math.BigInteger) value).longValue()); + } else if (value instanceof java.math.BigDecimal) { + constant = MySQLConstant.createIntConstant(((java.math.BigDecimal) value).longValue()); + } + + else if (value instanceof Float) { + constant = MySQLConstant.createDoubleConstant(Double.valueOf((Float) value)); + } else if (value instanceof Double) { + constant = MySQLConstant.createDoubleConstant((Double) value); + } + + else if (value instanceof String) { + constant = MySQLConstant.createStringConstant((String) value); + } + + else if (value instanceof Boolean) { + constant = MySQLConstant.createBoolean((Boolean) value); + } + + else if (value == null) { + constant = MySQLConstant.createNullConstant(); + } else { + throw new AssertionError(value.getClass().getName()); + } + List v = result.get(idxNameMap.get(i)); + v.add(constant); + } catch (SQLException e) { + System.out.println(e.getMessage()); + throw new IgnoreMeException(); + } + } + ++resultRows; + if (resultRows > 100) { + throw new IgnoreMeException(); + } + } + rs.close(); + Main.nrSuccessfulActions.addAndGet(1); + } catch (SQLException e) { + Main.nrUnsuccessfulActions.addAndGet(1); + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + state.getState().getLocalState().log(queryString); + throw new AssertionError(e.getMessage()); + } + } finally { + if (rs != null) { + rs.close(); + } + } + } finally { + if (stmt != null) { + stmt.close(); + } + } + return result; + } + + private MySQLTable genTemporaryTable(MySQLSelect select, String tableName) throws SQLException { + List fetchColumns = select.getFetchColumns(); + int columnNumber = fetchColumns.size(); + Map idxTypeMap = getColumnTypeFromSelect(select); + + List databaseColumns = new ArrayList<>(); + for (int i = 0; i < columnNumber; ++i) { + String columnName = "c" + String.valueOf(i); + MySQLColumn column = new MySQLColumn(columnName, idxTypeMap.get(i).getPrimitiveDataType(), false, 0); + databaseColumns.add(column); + } + MySQLTable table = new MySQLTable(tableName, databaseColumns, null, null); + for (MySQLColumn c : databaseColumns) { + c.setTable(table); + } + + return table; + } + + private MySQLTable createTemporaryTable(MySQLSelect select, String tableName, String valuesString) throws SQLException { + List fetchColumns = select.getFetchColumns(); + int columnNumber = fetchColumns.size(); + Map idxTypeMap = getColumnTypeFromSelect(select); + + StringBuilder sb = new StringBuilder(); + sb.append("CREATE TABLE " + tableName + " ("); + for (int i = 0; i < columnNumber; ++i) { + String columnTypeName = ""; + if (idxTypeMap.get(i) != null) { + columnTypeName = idxTypeMap.get(i).toString(); + } + sb.append("c" + String.valueOf(i) + " " + columnTypeName + ", "); + } + sb.deleteCharAt(sb.length() - 1); + sb.deleteCharAt(sb.length() - 1); + sb.append(");"); + String crateTableString = sb.toString(); + if (options.logEachSelect()) { + logger.writeCurrent(crateTableString); + } + Statement s = null; + try { + s = this.con.createStatement(); + s.execute(crateTableString); + } finally { + if (s != null) { + s.close(); + } + } + + String selectString = MySQLVisitor.asString(select); + StringBuilder sb2 = new StringBuilder(); + if (Randomly.getBoolean()) { + sb2.append("INSERT INTO " + tableName + " "+ selectString); + } else { + sb2.append("INSERT INTO " + tableName + " "+ valuesString); + } + + String insertValueString = sb2.toString(); + if (options.logEachSelect()) { + logger.writeCurrent(insertValueString); + } + s = null; + try { + s = this.con.createStatement(); + s.execute(insertValueString); + } finally { + s.close(); + } + + List databaseColumns = new ArrayList<>(); + for (int i = 0; i < columnNumber; ++i) { + String columnName = "c" + String.valueOf(i); + MySQLColumn column = new MySQLColumn(columnName, idxTypeMap.get(i).getPrimitiveDataType(), false, 0); + databaseColumns.add(column); + } + MySQLTable table = new MySQLTable(tableName, databaseColumns, null, null); + for (MySQLColumn c : databaseColumns) { + c.setTable(table); + } + + return table; + } + + private void dropTemporaryTable(String tableName) throws SQLException { + String dropString = "DROP TABLE " + tableName + ";"; + if (options.logEachSelect()) { + logger.writeCurrent(dropString); + } + Statement s = null; + try { + s = this.con.createStatement(); + s.execute(dropString); + } finally { + s.close(); + } + } + + private Map getColumnTypeFromSelect(MySQLSelect select) throws SQLException { + Map idxTypeMap = new HashMap<>(); + String tempTableName = "temp0"; + String tempTableCreate = "CREATE TABLE " + tempTableName + " AS " + MySQLVisitor.asString(select); + String tableDescribe = "DESCRIBE " + tempTableName; + if (options.logEachSelect()) { + logger.writeCurrent(tempTableCreate); + } + Statement s = null; + try { + s = this.con.createStatement(); + s.execute(tempTableCreate); + } catch(Exception e) { + if (e.getMessage().contains("Data truncation:")) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(e.getMessage()); + } + } finally { + s.close(); + } + Map> typeResult = null; + try { + typeResult = getQueryResult(tableDescribe, state); + } finally { + dropTemporaryTable(tempTableName); + } + if (typeResult == null) { + throw new AssertionError("can not get the return type of query"); + } + // the first column, c0, of typeResult is the name of column, + // the second column, c1, of typeResult is the type of column. + List types = typeResult.get("c1"); + for (int i = 0; i < types.size(); ++i) { + String typeName = ""; + if (types.get(i) instanceof MySQLConstant) { + MySQLConstant tString = (MySQLConstant) types.get(i); + typeName = tString.getTextRepresentation(); + } else { + throw new AssertionError(types.get(i).getClass().toString()); + } + + MySQLCompositeDataType cType = new MySQLCompositeDataType(typeName); + idxTypeMap.put(i, cType); + } + + return idxTypeMap; + } + + private boolean compareResult(Map> r1, Map> r2) { + if (r1.size() != r2.size()) { + return false; + } + for (Map.Entry < String, List > entry: r1.entrySet()) { + String currentKey = entry.getKey(); + if (!r2.containsKey(currentKey)) { + return false; + } + List v1= entry.getValue(); + List v2= r2.get(currentKey); + if (v1.size() != v2.size()) { + return false; + } + List v1Value = new ArrayList<>(v1.stream().map(c -> c.toString()).collect(Collectors.toList())); + List v2Value = new ArrayList<>(v2.stream().map(c -> c.toString()).collect(Collectors.toList())); + Collections.sort(v1Value); + Collections.sort(v2Value); + if (!v1Value.equals(v2Value)) { + return false; + } + } + return true; + } + + private MySQLSelect genSimpleSelect() throws SQLException { + MySQLTables tables = s.getRandomTableNonEmptyTables(); + tablesFromOuterContext = tables.getTables(); + List tableL = tables.getTables().stream().map(t -> new MySQLTableReference(t)) + .collect(Collectors.toList()); + MySQLExpressionGenerator exprGen = new MySQLExpressionGenerator(state).setColumns(tables.getColumns()); + this.foldedExpr = genCondition(exprGen, null, null); + + MySQLSelect select = new MySQLSelect(); + if (Randomly.getBoolean()) { + List joins = genJoinExpressions(tableL, state, null, null); + if (joins.size() > 0) { + select.setJoinClauses(joins); + this.joinsInExpr = joins; + } + } + select.setFromList(tableL); + + List fetchColumns = new ArrayList<>(); + int columnIdx = 0; + for (MySQLColumn c : tables.getColumns()) { + MySQLColumnReference cRef = new MySQLColumnReference(c, null); + MySQLAlias cAlias = new MySQLAlias(cRef, "c" + String.valueOf(columnIdx)); + fetchColumns.add(cAlias); + columnIdx++; + } + + // add the expression as last fetch column + MySQLAlias eAlias = new MySQLAlias(this.foldedExpr, "c" + String.valueOf(columnIdx)); + fetchColumns.add(eAlias); + + select.setFetchColumns(fetchColumns); + + originalQueryString = MySQLVisitor.asString(select); + + + Map> queryRes = null; + queryRes = getQueryResult(originalQueryString, state); + if (queryRes.get("c0").size() == 0) { + throw new IgnoreMeException(); + } + + // save the result first + selectResult.clear(); + selectResult.putAll(queryRes); + + // get the constant corresponding to each row from results + List summary = queryRes.remove("c" + String.valueOf(columnIdx)); + + Boolean emptyRes = queryRes.get(queryRes.keySet().iterator().next()).size() == 0; + + Map columnType = null; + if (!emptyRes) { + columnType = getColumnTypeFromSelect(select); + } + HashMap ct = new HashMap<>(); + LinkedHashMap> dbstate = new LinkedHashMap<>(); + // do not put the last fetch column to values + for (int i = 0; i < fetchColumns.size() - 1; ++i) { + MySQLAlias cAlias = (MySQLAlias) fetchColumns.get(i); + MySQLColumnReference cRef = (MySQLColumnReference) cAlias.getExpression(); + String columnName = cAlias.getAlias(); + dbstate.put(cRef, queryRes.get(columnName)); + ct.put(cRef, columnType.get(i)); + } + + foldedExpressionReturnType = columnType.get(fetchColumns.size() - 1); + + this.constantResOfFoldedExpr = new MySQLResultMap(dbstate, ct, summary, foldedExpressionReturnType); + + return select; + } + + private MySQLSelect genSelectWithCorrelatedSubquery() throws SQLException { + MySQLTables outerQueryRandomTables = s.getRandomTableNonEmptyTables(); + MySQLTables innerQueryRandomTables = s.getRandomTableNonEmptyTables(); + + List innerQueryFromTables = new ArrayList<>(); + for (MySQLTable t : innerQueryRandomTables.getTables()) { + if (!outerQueryRandomTables.isContained(t)) { + innerQueryFromTables.add(new MySQLTableReference(t)); + } + } + for (MySQLTable t : outerQueryRandomTables.getTables()) { + if (innerQueryRandomTables.isContained(t)) { + innerQueryRandomTables.removeTable(t); + + List newColumns = new ArrayList<>(); + for (MySQLColumn c : t.getColumns()) { + MySQLColumn newColumn = new MySQLColumn(c.getName(), c.getType(), false, 0); + newColumns.add(newColumn); + } + MySQLTable newTable = new MySQLTable(t.getName() + "a", newColumns, null, null); + for (MySQLColumn c : newColumns) { + c.setTable(newTable); + } + innerQueryRandomTables.addTable(newTable); + + MySQLAlias alias = new MySQLAlias(new MySQLTableReference(t), newTable.getName()); + innerQueryFromTables.add(alias); + } + } + + List innerQueryColumns = new ArrayList<>(); + innerQueryColumns.addAll(innerQueryRandomTables.getColumns()); + innerQueryColumns.addAll(outerQueryRandomTables.getColumns()); + gen = new MySQLExpressionGenerator(state).setColumns(innerQueryColumns); + + MySQLSelect innerQuery = new MySQLSelect(); + innerQuery.setFromList(innerQueryFromTables); + + MySQLExpression innerQueryWhereCondition = gen.generateBooleanExpression(); + innerQuery.setWhereClause(innerQueryWhereCondition); + + // use aggregate function in fetch column + MySQLColumnReference innerQueryAggr = new MySQLColumnReference(Randomly.fromList(innerQueryRandomTables.getColumns()), null); + MySQLAggregateFunction windowFunction = MySQLAggregateFunction.getRandom(); + MySQLExpression innerQueryAggrName = new MySQLAggregate(Arrays.asList(innerQueryAggr), windowFunction); + innerQuery.setFetchColumns(Arrays.asList(innerQueryAggrName)); + + this.foldedExpr = innerQuery; + + // outer query + MySQLSelect outerQuery = new MySQLSelect(); + List outerQueryFromTableRefs = outerQueryRandomTables.getTables().stream().map(t -> new MySQLTableReference(t)).collect(Collectors.toList()); + outerQuery.setFromList(outerQueryFromTableRefs); + tablesFromOuterContext = outerQueryRandomTables.getTables(); + + List fetchColumns = new ArrayList<>(); + int columnIdx = 0; + for (MySQLColumn c : outerQueryRandomTables.getColumns()) { + MySQLColumnReference cRef = new MySQLColumnReference(c, null); + MySQLAlias cAlias = new MySQLAlias(cRef, "c" + String.valueOf(columnIdx)); + fetchColumns.add(cAlias); + columnIdx++; + } + + // add the expression as last fetch column + MySQLAlias subqueryAlias = new MySQLAlias(innerQuery, "c" + String.valueOf(columnIdx)); + fetchColumns.add(subqueryAlias); + + outerQuery.setFetchColumns(fetchColumns); + + originalQueryString = MySQLVisitor.asString(outerQuery); + + + Map> queryRes = null; + try { + queryRes = getQueryResult(originalQueryString, state); + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + throw new AssertionError(e.getMessage()); + } + } + // just ignore the empty result, because of the empty table + if (queryRes.get("c0").size() == 0) { + throw new IgnoreMeException(); + } + + // save the result first + selectResult.clear(); + selectResult.putAll(queryRes); + + // get the constant corresponding to each row from results + List summary = queryRes.remove("c" + String.valueOf(columnIdx)); + + Boolean emptyRes = queryRes.get(queryRes.keySet().iterator().next()).size() == 0; + Map columnType = null; + if (!emptyRes) { + columnType = getColumnTypeFromSelect(outerQuery); + } + + LinkedHashMap> dbstate = new LinkedHashMap<>(); + HashMap ct = new HashMap<>(); + // do not put the last fetch column to values + for (int i = 0; i < fetchColumns.size() - 1; ++i) { + MySQLAlias cAlias = (MySQLAlias) fetchColumns.get(i); + MySQLColumnReference cRef = (MySQLColumnReference) cAlias.getExpression(); + String columnName = cAlias.getAlias(); + dbstate.put(cRef, queryRes.get(columnName)); + ct.put(cRef, columnType.get(i)); + } + + foldedExpressionReturnType = columnType.get(fetchColumns.size() - 1); + + this.constantResOfFoldedExpr = new MySQLResultMap(dbstate, ct, summary, foldedExpressionReturnType); + + return outerQuery; + } + + Boolean isEmptyTable(MySQLTable t) throws SQLException { + String queryString = "SELECT * FROM " + MySQLVisitor.asString(new MySQLTableReference(t)) + ";"; + int resultRows = 0; + Statement s = null; + try { + s = this.con.createStatement(); + ResultSet rs = null; + try { + rs = s.executeQuery(queryString); + while (rs.next()) { + ++resultRows; + } + rs.close(); + } catch (SQLException e) { + if (errors.errorIsExpected(e.getMessage())) { + throw new IgnoreMeException(); + } else { + state.getState().getLocalState().log(queryString); + throw new AssertionError(e.getMessage()); + } + } finally { + rs.close(); + } + } finally { + if (s != null) { + s.close(); + } + } + return resultRows == 0; + } + + public boolean useSubquery() { + if (this.state.getDbmsSpecificOptions().coddTestModel.equals("random")) { + return Randomly.getBoolean(); + } else if (this.state.getDbmsSpecificOptions().coddTestModel.equals("expression")) { + return false; + } else if (this.state.getDbmsSpecificOptions().coddTestModel.equals("subquery")) { + return true; + } else { + System.out.printf("Wrong option of --coddtest-model, should be one of: random, expression, subquery"); + System.exit(1); + return false; + } + } + + public boolean useCorrelatedSubquery() { + return Randomly.getBoolean(); + } + + @Override + public String getLastQueryString() { + return originalQueryString; + } + + @Override + public Reproducer getLastReproducer() { + return reproducer; + } +} From 412f69e603f407fc4ee2e200893fbe9b92ea69ab Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Tue, 4 Mar 2025 15:58:28 +0800 Subject: [PATCH 12/16] fix a typo --- .../cockroachdb/CockroachDBToStringVisitor.java | 8 ++++---- src/sqlancer/cockroachdb/CockroachDBVisitor.java | 8 ++++---- ...DBWithClasure.java => CockroachDBWithClause.java} | 4 ++-- .../oracle/CockroachDBCODDTestOracle.java | 8 ++++---- src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java | 8 ++++---- src/sqlancer/mysql/MySQLToStringVisitor.java | 6 +++--- src/sqlancer/mysql/oracle/MySQLCODDTestOracle.java | 8 ++++---- .../sqlite3/SQLite3ExpectedValueVisitor.java | 10 +++++----- src/sqlancer/sqlite3/SQLite3ToStringVisitor.java | 12 ++++++------ src/sqlancer/sqlite3/SQLite3Visitor.java | 8 ++++---- src/sqlancer/sqlite3/ast/SQLite3Expression.java | 4 ++-- src/sqlancer/sqlite3/ast/SQLite3Select.java | 12 ++++++------ .../sqlite3/oracle/SQLite3CODDTestOracle.java | 12 ++++++------ .../{TiDBWithClasure.java => TiDBWithClause.java} | 4 ++-- src/sqlancer/tidb/oracle/TiDBCODDTestOracle.java | 10 +++++----- src/sqlancer/tidb/visitor/TiDBToStringVisitor.java | 8 ++++---- src/sqlancer/tidb/visitor/TiDBVisitor.java | 8 ++++---- 17 files changed, 69 insertions(+), 69 deletions(-) rename src/sqlancer/cockroachdb/ast/{CockroachDBWithClasure.java => CockroachDBWithClause.java} (72%) rename src/sqlancer/tidb/ast/{TiDBWithClasure.java => TiDBWithClause.java} (75%) diff --git a/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java b/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java index 47e672731..81230bc22 100644 --- a/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java +++ b/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java @@ -29,7 +29,7 @@ import sqlancer.cockroachdb.ast.CockroachDBTableReference; import sqlancer.cockroachdb.ast.CockroachDBTypeof; import sqlancer.cockroachdb.ast.CockroachDBValues; -import sqlancer.cockroachdb.ast.CockroachDBWithClasure; +import sqlancer.cockroachdb.ast.CockroachDBWithClause; import sqlancer.common.visitor.ToStringVisitor; public class CockroachDBToStringVisitor extends ToStringVisitor implements CockroachDBVisitor { @@ -292,11 +292,11 @@ public void visit(CockroachDBValues values) { } @Override - public void visit(CockroachDBWithClasure withClasure) { + public void visit(CockroachDBWithClause withClause) { sb.append("WITH "); - visit(withClasure.getLeft()); + visit(withClause.getLeft()); sb.append(" AS ("); - visit(withClasure.getRight()); + visit(withClause.getRight()); sb.append(") "); } diff --git a/src/sqlancer/cockroachdb/CockroachDBVisitor.java b/src/sqlancer/cockroachdb/CockroachDBVisitor.java index d82c6c948..7d029129b 100644 --- a/src/sqlancer/cockroachdb/CockroachDBVisitor.java +++ b/src/sqlancer/cockroachdb/CockroachDBVisitor.java @@ -21,7 +21,7 @@ import sqlancer.cockroachdb.ast.CockroachDBTableReference; import sqlancer.cockroachdb.ast.CockroachDBTypeof; import sqlancer.cockroachdb.ast.CockroachDBValues; -import sqlancer.cockroachdb.ast.CockroachDBWithClasure; +import sqlancer.cockroachdb.ast.CockroachDBWithClause; public interface CockroachDBVisitor { @@ -51,7 +51,7 @@ public interface CockroachDBVisitor { void visit(CockroachDBExists existsExpr); void visit(CockroachDBExpressionBag exprBag); void visit(CockroachDBValues values); - void visit(CockroachDBWithClasure withClasure); + void visit(CockroachDBWithClause withClause); void visit(CockroachDBTableAndColumnReference tableAndColumnReference); void visit(CockroachDBAlias alias); void visit(CockroachDBTypeof typeOf); @@ -88,8 +88,8 @@ default void visit(CockroachDBExpression expr) { visit((CockroachDBExpressionBag) expr); } else if (expr instanceof CockroachDBValues) { visit((CockroachDBValues) expr); - } else if (expr instanceof CockroachDBWithClasure) { - visit ((CockroachDBWithClasure) expr); + } else if (expr instanceof CockroachDBWithClause) { + visit ((CockroachDBWithClause) expr); } else if (expr instanceof CockroachDBTableAndColumnReference) { visit ((CockroachDBTableAndColumnReference) expr); } else if (expr instanceof CockroachDBAlias) { diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBWithClasure.java b/src/sqlancer/cockroachdb/ast/CockroachDBWithClause.java similarity index 72% rename from src/sqlancer/cockroachdb/ast/CockroachDBWithClasure.java rename to src/sqlancer/cockroachdb/ast/CockroachDBWithClause.java index e33a22935..0f2f62e05 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBWithClasure.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBWithClause.java @@ -1,11 +1,11 @@ package sqlancer.cockroachdb.ast; -public class CockroachDBWithClasure implements CockroachDBExpression { +public class CockroachDBWithClause implements CockroachDBExpression { private CockroachDBExpression left; private CockroachDBExpression right; - public CockroachDBWithClasure(CockroachDBExpression left, CockroachDBExpression right) { + public CockroachDBWithClause(CockroachDBExpression left, CockroachDBExpression right) { this.left = left; this.right = right; } diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBCODDTestOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBCODDTestOracle.java index 5c1a5e61f..56544a1d1 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBCODDTestOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBCODDTestOracle.java @@ -51,7 +51,7 @@ import sqlancer.cockroachdb.ast.CockroachDBTableReference; import sqlancer.cockroachdb.ast.CockroachDBTypeof; import sqlancer.cockroachdb.ast.CockroachDBValues; -import sqlancer.cockroachdb.ast.CockroachDBWithClasure; +import sqlancer.cockroachdb.ast.CockroachDBWithClause; import sqlancer.cockroachdb.ast.CockroachDBAggregate.CockroachDBAggregateFunction; import sqlancer.cockroachdb.ast.CockroachDBBinaryComparisonOperator.CockroachDBComparisonOperator; import sqlancer.cockroachdb.ast.CockroachDBBinaryLogicalOperation.CockroachDBBinaryLogicalOperator; @@ -314,16 +314,16 @@ else if (auxiliaryQueryResult.size() == 1 && Randomly.getBoolean()) { CockroachDBExpressionBag tempTableRefBag = new CockroachDBExpressionBag(tempTableRef); CockroachDBTableAndColumnReference tableAndColumnRef = new CockroachDBTableAndColumnReference(temporaryTable); - CockroachDBWithClasure withClasure = new CockroachDBWithClasure(tableAndColumnRef, auxiliaryQuery); + CockroachDBWithClause withClause = new CockroachDBWithClause(tableAndColumnRef, auxiliaryQuery); originalQuery = genSelectExpression(tempTableRefBag, null, null); - originalQuery.setWithClause(withClasure); + originalQuery.setWithClause(withClause); originalQueryString = CockroachDBVisitor.asString(originalQuery); originalResult = getQueryResult(originalQueryString, state); // folded query if (Randomly.getBoolean()) { // folded query: WITH table AS VALUES () - withClasure.updateRight(values); + withClause.updateRight(values); foldedQueryString = CockroachDBVisitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); } else if (Randomly.getBoolean()) { diff --git a/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java b/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java index 414f6ee4e..41297effe 100644 --- a/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java +++ b/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java @@ -220,8 +220,8 @@ else if (auxiliaryQueryResult.size() == 1 && Randomly.getBoolean()) { DuckDBExpressionBag tableBag = new DuckDBExpressionBag(tableRef); originalQuery = this.genSelectExpression(tableBag, null); - DuckDBWithClause optWithClasure = new DuckDBWithClause(tableRef, Arrays.asList(auxiliaryQuery)); - originalQuery.setWithClause(optWithClasure); + DuckDBWithClause optWithClause = new DuckDBWithClause(tableRef, Arrays.asList(auxiliaryQuery)); + originalQuery.setWithClause(optWithClause); originalQueryString = DuckDBToStringVisitor.asString(originalQuery); originalResult = getQueryResult(originalQueryString, state); @@ -239,8 +239,8 @@ else if (auxiliaryQueryResult.size() == 1 && Randomly.getBoolean()) { DuckDBValues valueRow = new DuckDBValues(rowRs); values.add(valueRow); } - DuckDBWithClause refWithClasure = new DuckDBWithClause(tableRef, values); - originalQuery.setWithClause(refWithClasure); + DuckDBWithClause refWithClause = new DuckDBWithClause(tableRef, values); + originalQuery.setWithClause(refWithClause); foldedQueryString = DuckDBToStringVisitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); } else if (Randomly.getBoolean()) { diff --git a/src/sqlancer/mysql/MySQLToStringVisitor.java b/src/sqlancer/mysql/MySQLToStringVisitor.java index fd8667208..c22a29e56 100644 --- a/src/sqlancer/mysql/MySQLToStringVisitor.java +++ b/src/sqlancer/mysql/MySQLToStringVisitor.java @@ -405,11 +405,11 @@ public void visit(MySQLValuesRow values) { } @Override - public void visit(MySQLWithClause withClasure) { + public void visit(MySQLWithClause withClause) { sb.append("WITH "); - visit(withClasure.getLeft()); + visit(withClause.getLeft()); sb.append(" AS ("); - visit(withClasure.getRight()); + visit(withClause.getRight()); sb.append(") "); } diff --git a/src/sqlancer/mysql/oracle/MySQLCODDTestOracle.java b/src/sqlancer/mysql/oracle/MySQLCODDTestOracle.java index 5facf62c2..b75f35d01 100644 --- a/src/sqlancer/mysql/oracle/MySQLCODDTestOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLCODDTestOracle.java @@ -281,14 +281,14 @@ else if (auxiliaryQueryResult.size() == 1 && Randomly.getBoolean()) { MySQLExpressionBag tempTableRefBag = new MySQLExpressionBag(tempTableRef); MySQLTableAndColumnReference tableAndColumnRef = new MySQLTableAndColumnReference(temporaryTable); - MySQLWithClause withClasure = null; + MySQLWithClause withClause = null; if (Randomly.getBoolean()) { - withClasure = new MySQLWithClause(tableAndColumnRef, auxiliaryQuery); + withClause = new MySQLWithClause(tableAndColumnRef, auxiliaryQuery); } else { - withClasure = new MySQLWithClause(tableAndColumnRef, resValues); + withClause = new MySQLWithClause(tableAndColumnRef, resValues); } originalQuery = genSelectExpression(tempTableRefBag, null, null); - originalQuery.setWithClause(withClasure); + originalQuery.setWithClause(withClause); originalQueryString = MySQLVisitor.asString(originalQuery); originalResult = getQueryResult(originalQueryString, state); diff --git a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java index 246105a5e..594fe5c92 100644 --- a/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ExpectedValueVisitor.java @@ -28,7 +28,7 @@ import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClasure; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClause; import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Subquery; import sqlancer.sqlite3.ast.SQLite3Expression.TypeLiteral; @@ -318,10 +318,10 @@ public void visit(SQLite3Alias alias) { } @Override - public void visit(SQLite3WithClasure withClasure) { - print(withClasure); - print(withClasure.getLeft()); - print(withClasure.getRight()); + public void visit(SQLite3WithClause withClause) { + print(withClause); + print(withClause.getLeft()); + print(withClause.getRight()); } @Override diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index a7b392ea7..ebe9f88a5 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -36,7 +36,7 @@ import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClasure; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClause; import sqlancer.sqlite3.ast.SQLite3Expression.Subquery; import sqlancer.sqlite3.ast.SQLite3Expression.TypeLiteral; import sqlancer.sqlite3.ast.SQLite3Function; @@ -112,8 +112,8 @@ public void visit(SQLite3Select s, boolean inner) { if (inner) { sb.append("("); } - if (s.getWithClasure() != null) { - visit(s.getWithClasure()); + if (s.getWithClause() != null) { + visit(s.getWithClause()); sb.append(" "); } sb.append("SELECT "); @@ -519,11 +519,11 @@ public void visit(SQLite3Alias alias) { } @Override - public void visit(SQLite3WithClasure withClasure) { + public void visit(SQLite3WithClause withClause) { sb.append("WITH "); - visit(withClasure.getLeft()); + visit(withClause.getLeft()); sb.append(" AS "); - visit(withClasure.getRight()); + visit(withClause.getRight()); } @Override diff --git a/src/sqlancer/sqlite3/SQLite3Visitor.java b/src/sqlancer/sqlite3/SQLite3Visitor.java index d5ebce096..5e1e5dbf6 100644 --- a/src/sqlancer/sqlite3/SQLite3Visitor.java +++ b/src/sqlancer/sqlite3/SQLite3Visitor.java @@ -27,7 +27,7 @@ import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Text; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClasure; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClause; import sqlancer.sqlite3.ast.SQLite3Expression.Sqlite3BinaryOperation; import sqlancer.sqlite3.ast.SQLite3Expression.Subquery; import sqlancer.sqlite3.ast.SQLite3Expression.TypeLiteral; @@ -140,7 +140,7 @@ default void visit(SQLite3PostfixUnaryOperation exp) { void visit(SQLite3Alias alias); - void visit(SQLite3WithClasure withClasure); + void visit(SQLite3WithClause withClause); void visit(SQLite3TableAndColumnRef tableAndColumnRef); @@ -217,8 +217,8 @@ default void visit(SQLite3Expression expr) { visit((SQLite3SetClause) expr); } else if (expr instanceof SQLite3Alias) { visit((SQLite3Alias) expr); - } else if (expr instanceof SQLite3WithClasure) { - visit((SQLite3WithClasure) expr); + } else if (expr instanceof SQLite3WithClause) { + visit((SQLite3WithClause) expr); } else if (expr instanceof SQLite3TableAndColumnRef) { visit((SQLite3TableAndColumnRef) expr); } else if (expr instanceof SQLite3Values) { diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index 6c4b2f9ff..12e5c71f1 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -1563,12 +1563,12 @@ public boolean omitBracketsWhenPrinting() { } } - public static class SQLite3WithClasure extends SQLite3Expression { + public static class SQLite3WithClause extends SQLite3Expression { private SQLite3Expression left; private SQLite3Expression right; - public SQLite3WithClasure(SQLite3Expression left, SQLite3Expression right) { + public SQLite3WithClause(SQLite3Expression left, SQLite3Expression right) { this.left = left; this.right = right; } diff --git a/src/sqlancer/sqlite3/ast/SQLite3Select.java b/src/sqlancer/sqlite3/ast/SQLite3Select.java index 042aab856..6f1571df0 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Select.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Select.java @@ -25,7 +25,7 @@ public class SQLite3Select extends SQLite3Expression private List fetchColumns = Collections.emptyList(); private List joinStatements = Collections.emptyList(); private SQLite3Expression havingClause; - private SQLite3WithClasure withClause = null; + private SQLite3WithClause withClause = null; public SQLite3Select() { } @@ -165,15 +165,15 @@ public String asString() { return SQLite3Visitor.asString(this); } - public void setWithClasure(SQLite3WithClasure withClasure) { - this.withClause = withClasure; + public void setWithClause(SQLite3WithClause withClause) { + this.withClause = withClause; } - public void updateWithClasureRight(SQLite3Expression withClasureRight) { - this.withClause.updateRight(withClasureRight); + public void updateWithClauseRight(SQLite3Expression withClauseRight) { + this.withClause.updateRight(withClauseRight); } - public SQLite3Expression getWithClasure() { + public SQLite3Expression getWithClause() { return this.withClause; } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java index c4e8d892a..17444026a 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java @@ -41,7 +41,7 @@ import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3ResultMap; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Typeof; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3Values; -import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClasure; +import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3WithClause; import sqlancer.sqlite3.ast.SQLite3Expression.Join.JoinType; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm.Ordering; import sqlancer.sqlite3.ast.SQLite3Expression.SQLite3OrderingTerm; @@ -202,8 +202,8 @@ else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowPro SQLite3Table temporaryTable = this.genTemporaryTable(auxiliaryQuery, this.tempTableName); originalQuery = this.genSelectExpression(temporaryTable, null); SQLite3TableAndColumnRef tableAndColumnRef = new SQLite3TableAndColumnRef(temporaryTable); - SQLite3WithClasure withClasure = new SQLite3WithClasure(tableAndColumnRef, new SQLite3Select(auxiliaryQuery)); - originalQuery.setWithClasure(withClasure); + SQLite3WithClause withClause = new SQLite3WithClause(tableAndColumnRef, new SQLite3Select(auxiliaryQuery)); + originalQuery.setWithClause(withClause); originalQueryString = SQLite3Visitor.asString(originalQuery); originalResult = getQueryResult(originalQueryString, state); // folded query @@ -212,13 +212,13 @@ else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowPro // common table expression // folded query: WITH table AS VALUES () SQLite3Values values = new SQLite3Values(auxiliaryQueryResult, temporaryTable.getColumns()); - originalQuery.updateWithClasureRight(values); + originalQuery.updateWithClauseRight(values); foldedQueryString = SQLite3Visitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); } else if (Randomly.getBoolean() && this.testDerivedTable()) { // derived table // folded query: SELECT FROM () AS table - originalQuery.setWithClasure(null); + originalQuery.setWithClause(null); SQLite3TableReference tempTableRef = new SQLite3TableReference(temporaryTable); SQLite3Alias alias = new SQLite3Alias(new SQLite3Select(auxiliaryQuery), tempTableRef); originalQuery.replaceFromTable(this.tempTableName, alias); @@ -229,7 +229,7 @@ else if (auxiliaryQueryResult.size() == 1 && Randomly.getBooleanWithRatherLowPro // folded query: CREATE the table and INSERT INTO table subquery try { this.createTemporaryTable(auxiliaryQuery, this.tempTableName); - originalQuery.setWithClasure(null); + originalQuery.setWithClause(null); foldedQueryString = SQLite3Visitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); } finally { diff --git a/src/sqlancer/tidb/ast/TiDBWithClasure.java b/src/sqlancer/tidb/ast/TiDBWithClause.java similarity index 75% rename from src/sqlancer/tidb/ast/TiDBWithClasure.java rename to src/sqlancer/tidb/ast/TiDBWithClause.java index ce3d650eb..a06dea6b3 100644 --- a/src/sqlancer/tidb/ast/TiDBWithClasure.java +++ b/src/sqlancer/tidb/ast/TiDBWithClause.java @@ -1,10 +1,10 @@ package sqlancer.tidb.ast; -public class TiDBWithClasure implements TiDBExpression { +public class TiDBWithClause implements TiDBExpression { private TiDBExpression left; private TiDBExpression right; - public TiDBWithClasure(TiDBExpression left, TiDBExpression right) { + public TiDBWithClause(TiDBExpression left, TiDBExpression right) { this.left = left; this.right = right; } diff --git a/src/sqlancer/tidb/oracle/TiDBCODDTestOracle.java b/src/sqlancer/tidb/oracle/TiDBCODDTestOracle.java index 88df9dc22..7b5e6abde 100644 --- a/src/sqlancer/tidb/oracle/TiDBCODDTestOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBCODDTestOracle.java @@ -52,7 +52,7 @@ import sqlancer.tidb.ast.TiDBResultMap; import sqlancer.tidb.ast.TiDBValues; import sqlancer.tidb.ast.TiDBValuesRow; -import sqlancer.tidb.ast.TiDBWithClasure; +import sqlancer.tidb.ast.TiDBWithClause; import sqlancer.tidb.ast.TiDBAggregate.TiDBAggregateFunction; import sqlancer.tidb.ast.TiDBBinaryComparisonOperation.TiDBComparisonOperator; import sqlancer.tidb.ast.TiDBBinaryLogicalOperation.TiDBBinaryLogicalOperator; @@ -286,15 +286,15 @@ else if (auxiliaryQueryResult.size() == 1 && Randomly.getBoolean()) { TiDBExpressionBag tempTableRefBag = new TiDBExpressionBag(tempTableRef); TiDBTableAndColumnReference tableAndColumnRef = new TiDBTableAndColumnReference(temporaryTable); - TiDBWithClasure withClasure = null; + TiDBWithClause withClause = null; if (Randomly.getBoolean() || true) { - withClasure = new TiDBWithClasure(tableAndColumnRef, auxiliaryQuery); + withClause = new TiDBWithClause(tableAndColumnRef, auxiliaryQuery); } else { // there is an error in `WITH t0(c0) AS VALUES` - withClasure = new TiDBWithClasure(tableAndColumnRef, resValues); + withClause = new TiDBWithClause(tableAndColumnRef, resValues); } originalQuery = genSelectExpression(tempTableRefBag, null, null); - originalQuery.setWithClause(withClasure); + originalQuery.setWithClause(withClause); originalQueryString = TiDBVisitor.asString(originalQuery); originalResult = getQueryResult(originalQueryString, state); diff --git a/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java b/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java index ad1d762e0..a6c12a327 100644 --- a/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java +++ b/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java @@ -32,7 +32,7 @@ import sqlancer.tidb.ast.TiDBText; import sqlancer.tidb.ast.TiDBValues; import sqlancer.tidb.ast.TiDBValuesRow; -import sqlancer.tidb.ast.TiDBWithClasure; +import sqlancer.tidb.ast.TiDBWithClause; public class TiDBToStringVisitor extends ToStringVisitor implements TiDBVisitor { @@ -298,11 +298,11 @@ public void visit(TiDBValuesRow values) { } @Override - public void visit(TiDBWithClasure withClasure) { + public void visit(TiDBWithClause withClause) { sb.append("WITH "); - visit(withClasure.getLeft()); + visit(withClause.getLeft()); sb.append(" AS ("); - visit(withClasure.getRight()); + visit(withClause.getRight()); sb.append(") "); } diff --git a/src/sqlancer/tidb/visitor/TiDBVisitor.java b/src/sqlancer/tidb/visitor/TiDBVisitor.java index 847d7382f..ae9679483 100644 --- a/src/sqlancer/tidb/visitor/TiDBVisitor.java +++ b/src/sqlancer/tidb/visitor/TiDBVisitor.java @@ -21,7 +21,7 @@ import sqlancer.tidb.ast.TiDBText; import sqlancer.tidb.ast.TiDBValues; import sqlancer.tidb.ast.TiDBValuesRow; -import sqlancer.tidb.ast.TiDBWithClasure; +import sqlancer.tidb.ast.TiDBWithClause; public interface TiDBVisitor { @@ -60,8 +60,8 @@ default void visit(TiDBExpression expr) { visit((TiDBValues) expr); } else if (expr instanceof TiDBValuesRow) { visit((TiDBValuesRow) expr); - } else if (expr instanceof TiDBWithClasure) { - visit((TiDBWithClasure) expr); + } else if (expr instanceof TiDBWithClause) { + visit((TiDBWithClause) expr); } else if (expr instanceof TiDBResultMap) { visit((TiDBResultMap) expr); } else if (expr instanceof TiDBAllOperator) { @@ -101,7 +101,7 @@ default void visit(TiDBExpression expr) { void visit(TiDBTableAndColumnReference tAndCRef); void visit(TiDBValues values); void visit(TiDBValuesRow values); - void visit(TiDBWithClasure withClasure); + void visit(TiDBWithClause withClause); void visit(TiDBResultMap tableSummary); void visit(TiDBAllOperator allOperation); void visit(TiDBAnyOperator anyOperation); From 2d74254e65f85115e332e65cb0af2ccec162742b Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Tue, 4 Mar 2025 16:12:08 +0800 Subject: [PATCH 13/16] add comments for ExpressionBag --- src/sqlancer/cockroachdb/ast/CockroachDBExpressionBag.java | 4 ++++ src/sqlancer/duckdb/ast/DuckDBExpressionBag.java | 4 ++++ src/sqlancer/mysql/ast/MySQLExpressionBag.java | 5 +++++ src/sqlancer/sqlite3/ast/SQLite3Expression.java | 4 ++++ src/sqlancer/tidb/ast/TiDBExpressionBag.java | 3 +++ 5 files changed, 20 insertions(+) diff --git a/src/sqlancer/cockroachdb/ast/CockroachDBExpressionBag.java b/src/sqlancer/cockroachdb/ast/CockroachDBExpressionBag.java index 1289f2b00..03566a42b 100644 --- a/src/sqlancer/cockroachdb/ast/CockroachDBExpressionBag.java +++ b/src/sqlancer/cockroachdb/ast/CockroachDBExpressionBag.java @@ -1,5 +1,9 @@ package sqlancer.cockroachdb.ast; + +// The ExpressionBag is not a built-in SQL feature, +// but rather a utility class used in CODDTest's oracle construction +// to substitute expressions with their corresponding constant values. public class CockroachDBExpressionBag implements CockroachDBExpression { private CockroachDBExpression innerExpr; diff --git a/src/sqlancer/duckdb/ast/DuckDBExpressionBag.java b/src/sqlancer/duckdb/ast/DuckDBExpressionBag.java index 29e31cb19..4572a466a 100644 --- a/src/sqlancer/duckdb/ast/DuckDBExpressionBag.java +++ b/src/sqlancer/duckdb/ast/DuckDBExpressionBag.java @@ -1,5 +1,9 @@ package sqlancer.duckdb.ast; + +// The ExpressionBag is not a built-in SQL feature, +// but rather a utility class used in CODDTest's oracle construction +// to substitute expressions with their corresponding constant values. public class DuckDBExpressionBag implements DuckDBExpression { DuckDBExpression expr; diff --git a/src/sqlancer/mysql/ast/MySQLExpressionBag.java b/src/sqlancer/mysql/ast/MySQLExpressionBag.java index e39b1e99a..0e5e5f6f9 100644 --- a/src/sqlancer/mysql/ast/MySQLExpressionBag.java +++ b/src/sqlancer/mysql/ast/MySQLExpressionBag.java @@ -1,5 +1,10 @@ package sqlancer.mysql.ast; + + +// The ExpressionBag is not a built-in SQL feature, +// but rather a utility class used in CODDTest's oracle construction +// to substitute expressions with their corresponding constant values. public class MySQLExpressionBag implements MySQLExpression { private MySQLExpression innerExpr; diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index 12e5c71f1..f3542d863 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -1671,6 +1671,10 @@ public SQLite3CollateSequence getExplicitCollateSequence() { } } + + // The ExpressionBag is not a built-in SQL feature, + // but rather a utility class used in CODDTest's oracle construction + // to substitute expressions with their corresponding constant values. public static class SQLite3ExpressionBag extends SQLite3Expression { private SQLite3Expression innerExpr; diff --git a/src/sqlancer/tidb/ast/TiDBExpressionBag.java b/src/sqlancer/tidb/ast/TiDBExpressionBag.java index 2668f74fe..303952b9c 100644 --- a/src/sqlancer/tidb/ast/TiDBExpressionBag.java +++ b/src/sqlancer/tidb/ast/TiDBExpressionBag.java @@ -1,5 +1,8 @@ package sqlancer.tidb.ast; +// The ExpressionBag is not a built-in SQL feature, +// but rather a utility class used in CODDTest's oracle construction +// to substitute expressions with their corresponding constant values. public class TiDBExpressionBag implements TiDBExpression { private TiDBExpression innerExpr; From 9db8ae030c185ff619c50558c31a1cdb5a2f06bd Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Tue, 4 Mar 2025 20:39:56 +0800 Subject: [PATCH 14/16] remove deleteCharAt to simplify code logic --- .../CockroachDBToStringVisitor.java | 24 ++++--- .../oracle/CockroachDBCODDTestOracle.java | 9 ++- .../duckdb/DuckDBToStringVisitor.java | 11 +-- .../duckdb/test/DuckDBCODDTestOracle.java | 7 +- src/sqlancer/mysql/MySQLToStringVisitor.java | 68 ++++++++++-------- .../mysql/oracle/MySQLCODDTestOracle.java | 7 +- .../sqlite3/SQLite3ToStringVisitor.java | 34 +++++---- .../sqlite3/ast/SQLite3Expression.java | 8 ++- .../sqlite3/oracle/SQLite3CODDTestOracle.java | 7 +- .../tidb/oracle/TiDBCODDTestOracle.java | 7 +- .../tidb/visitor/TiDBToStringVisitor.java | 71 ++++++++++--------- 11 files changed, 137 insertions(+), 116 deletions(-) diff --git a/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java b/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java index 81230bc22..59e6de139 100644 --- a/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java +++ b/src/sqlancer/cockroachdb/CockroachDBToStringVisitor.java @@ -273,21 +273,24 @@ public void visit(CockroachDBValues values) { sb.append("(VALUES "); for (int i = 0; i < size; i++) { sb.append("("); + boolean isFirstColumn = true; for (CockroachDBColumn c: vs.keySet()) { + if (!isFirstColumn) { + sb.append(", "); + } sb.append(vs.get(c).get(i).toString()); if (!(vs.get(c).get(i) instanceof CockroachDBNullConstant)) { if (c.getType() != null) { sb.append("::" + c.getType().toString()); } } + isFirstColumn = false; + } + sb.append(")"); + if (i < size - 1) { sb.append(", "); } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.append("), "); } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); sb.append(")"); } @@ -343,7 +346,11 @@ public void visit(CockroachDBResultMap expr) { sb.append(" CASE "); for (int i = 0; i < size; i++) { sb.append("WHEN "); + Boolean isFirstCondition = true; for (CockroachDBColumnReference columnRef : dbstate.keySet()) { + if (!isFirstCondition) { + sb.append(" AND "); + } visit(columnRef); if (dbstate.get(columnRef).get(i) instanceof CockroachDBNullConstant) { sb.append(" IS NULL"); @@ -356,12 +363,9 @@ public void visit(CockroachDBResultMap expr) { } } } - sb.append(" AND "); - } - for (int k = 0; k < 4; k++) { - sb.deleteCharAt(sb.length() - 1); + isFirstCondition = false; } - sb.append("THEN "); + sb.append(" THEN "); visit(result.get(i)); if (!(result.get(i) instanceof CockroachDBNullConstant)) { if (expr.getResultType() != null) { diff --git a/src/sqlancer/cockroachdb/oracle/CockroachDBCODDTestOracle.java b/src/sqlancer/cockroachdb/oracle/CockroachDBCODDTestOracle.java index 56544a1d1..e9b3459a5 100644 --- a/src/sqlancer/cockroachdb/oracle/CockroachDBCODDTestOracle.java +++ b/src/sqlancer/cockroachdb/oracle/CockroachDBCODDTestOracle.java @@ -749,12 +749,11 @@ private CockroachDBTable createTemporaryTable(CockroachDBSelect select, String t if (idxTypeMap.get(i) != null) { columnTypeName = idxTypeMap.get(i).toString(); } - sb.append("c" + String.valueOf(i) + " " + columnTypeName + ", "); - // sb.append("c" + String.valueOf(i) + " " + idxTypeMap.get(i+1) + ", "); - // sb.append("c" + String.valueOf(i) + ", "); + sb.append("c" + String.valueOf(i) + " " + columnTypeName); + if (i < columnNumber - 1) { + sb.append(", "); + } } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); sb.append(");"); String crateTableString = sb.toString(); if (options.logEachSelect()) { diff --git a/src/sqlancer/duckdb/DuckDBToStringVisitor.java b/src/sqlancer/duckdb/DuckDBToStringVisitor.java index c8e21686c..c1e1af8b5 100644 --- a/src/sqlancer/duckdb/DuckDBToStringVisitor.java +++ b/src/sqlancer/duckdb/DuckDBToStringVisitor.java @@ -123,7 +123,11 @@ public void visit(DuckDBResultMap expr) { sb.append(" CASE "); for (int i = 0; i < size; i++) { sb.append("WHEN "); + Boolean isFirstCondition = true; for (DuckDBColumnReference columnRef : dbstate.keySet()) { + if (!isFirstCondition) { + sb.append(" AND "); + } visit(columnRef); if (dbstate.get(columnRef).get(i) instanceof DuckDBNullConstant) { sb.append(" IS NULL"); @@ -139,12 +143,9 @@ public void visit(DuckDBResultMap expr) { sb.append(" = "); visit(dbstate.get(columnRef).get(i)); } - sb.append(" AND "); - } - for (int k = 0; k < 4; k++) { - sb.deleteCharAt(sb.length() - 1); + isFirstCondition = false; } - sb.append("THEN "); + sb.append(" THEN "); visit(result.get(i)); sb.append(" "); } diff --git a/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java b/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java index 41297effe..0e835e47c 100644 --- a/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java +++ b/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java @@ -781,10 +781,11 @@ private DuckDBTable createTemporaryTable(DuckDBSelect select, String tableName) sb.append("CREATE TABLE " + tableName + " ("); for (int i = 0; i < columnNumber; ++i) { String columnTypeName = idxTypeMap.get(i).toString(); - sb.append("c" + String.valueOf(i) + " " + columnTypeName + ", "); + sb.append("c" + String.valueOf(i) + " " + columnTypeName); + if (i < columnNumber - 1) { + sb.append(", "); + } } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); sb.append(");"); String crateTableString = sb.toString(); if (options.logEachSelect()) { diff --git a/src/sqlancer/mysql/MySQLToStringVisitor.java b/src/sqlancer/mysql/MySQLToStringVisitor.java index c22a29e56..fce0afe25 100644 --- a/src/sqlancer/mysql/MySQLToStringVisitor.java +++ b/src/sqlancer/mysql/MySQLToStringVisitor.java @@ -371,16 +371,19 @@ public void visit(MySQLValues values) { // sb.append("("); for (int i = 0; i < size; i++) { sb.append("("); + Boolean isFirstColumn = true; for (MySQLColumn name : vs.keySet()) { + if (!isFirstColumn) { + sb.append(", "); + } visit(vs.get(name).get(i)); + isFirstColumn = false; + } + sb.append(")"); + if (i < size - 1) { sb.append(", "); } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.append("), "); } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); } @Override @@ -391,16 +394,19 @@ public void visit(MySQLValuesRow values) { sb.append("(VALUES "); for (int i = 0; i < size; i++) { sb.append("ROW("); + Boolean isFirstColumn = true; for (MySQLColumn name : vs.keySet()) { + if (!isFirstColumn) { + sb.append(", "); + } visit(vs.get(name).get(i)); + isFirstColumn = false; + } + sb.append(")"); + if (i < size - 1) { sb.append(", "); } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.append("), "); } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); sb.append(")"); } @@ -423,15 +429,15 @@ public void visit(MySQLResultMap tableSummary) { int size = vs.get(vs.keySet().iterator().next()).size(); if (size == 0) { sb.append("("); + Boolean isFirstColumn = true; for (MySQLColumnReference tr: vs.keySet()) { + if (!isFirstColumn) { + sb.append(" AND "); + } visit(tr); - sb.append(" IS NULL AND "); + sb.append(" IS NULL"); + isFirstColumn = false; } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); sb.append(")"); return; } @@ -439,7 +445,11 @@ public void visit(MySQLResultMap tableSummary) { sb.append(" CASE "); for (int i = 0; i < size; i++) { sb.append("WHEN "); + Boolean isFirstColumn = true; for (MySQLColumnReference tr: vs.keySet()) { + if (!isFirstColumn) { + sb.append(" AND "); + } visit(tr); if (vs.get(tr).get(i) instanceof MySQLNullConstant) { sb.append(" IS NULL"); @@ -453,13 +463,9 @@ public void visit(MySQLResultMap tableSummary) { sb.append(", " + columnType.get(tr).toString().replaceAll("'", "").replaceAll("\"", "") + ")"); } } - sb.append(" AND "); + isFirstColumn = false; } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.append("THEN "); + sb.append(" THEN "); sb.append("CONVERT("); visit(results.get(i)); sb.append(", " + tableSummary.getResultType().toString().replaceAll("'", "").replaceAll("\"", "") + ")"); @@ -485,13 +491,14 @@ public void visit(MySQLAllOperator allOperation) { } else { sb.append(" SELECT "); } - + Boolean isFirstColumn = true; for (MySQLColumn name : vs.keySet()) { + if (!isFirstColumn) { + sb.append(", "); + } visit(vs.get(name).get(i)); - sb.append(", "); + isFirstColumn = false; } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); } } else { visit(allOperation.getRightExpr()); @@ -516,13 +523,14 @@ public void visit(MySQLAnyOperator anyOperation) { } else { sb.append(" SELECT "); } - + Boolean isFirstColumn = true; for (MySQLColumn name : vs.keySet()) { + if (!isFirstColumn) { + sb.append(", "); + } visit(vs.get(name).get(i)); - sb.append(", "); + isFirstColumn = false; } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); } } else { visit(anyOperation.getRightExpr()); diff --git a/src/sqlancer/mysql/oracle/MySQLCODDTestOracle.java b/src/sqlancer/mysql/oracle/MySQLCODDTestOracle.java index b75f35d01..c6c9673a1 100644 --- a/src/sqlancer/mysql/oracle/MySQLCODDTestOracle.java +++ b/src/sqlancer/mysql/oracle/MySQLCODDTestOracle.java @@ -642,10 +642,11 @@ private MySQLTable createTemporaryTable(MySQLSelect select, String tableName, St if (idxTypeMap.get(i) != null) { columnTypeName = idxTypeMap.get(i).toString(); } - sb.append("c" + String.valueOf(i) + " " + columnTypeName + ", "); + sb.append("c" + String.valueOf(i) + " " + columnTypeName); + if (i < columnNumber - 1) { + sb.append(", "); + } } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); sb.append(");"); String crateTableString = sb.toString(); if (options.logEachSelect()) { diff --git a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java index ebe9f88a5..ccffb387f 100644 --- a/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java +++ b/src/sqlancer/sqlite3/SQLite3ToStringVisitor.java @@ -539,7 +539,11 @@ public void visit(SQLite3Values values) { sb.append("(VALUES "); for (int i = 0; i < size; i++) { sb.append("("); + Boolean isFirstColumn = true; for (String name : columnNames) { + if (!isFirstColumn) { + sb.append(", "); + } if (vs.get(name).get(i).getDataType() == SQLite3DataType.NULL) { visit(vs.get(name).get(i)); } else { @@ -563,15 +567,13 @@ public void visit(SQLite3Values values) { throw new IgnoreMeException(); } } - + isFirstColumn = true; + } + sb.append(")"); + if (i < size - 1) { sb.append(", "); } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.append("), "); } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); sb.append(")"); } @@ -600,13 +602,11 @@ public void visit(SQLite3ResultMap tableSummary) { sb.append("("); for (int j = 0; j < columnRefs.size(); ++j) { visit(columnRefs.get(j)); - sb.append(" IS NULL AND "); + sb.append(" IS NULL"); + if (j < columnRefs.size() - 1) { + sb.append(" AND "); + } } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); sb.append(")"); return; } @@ -622,13 +622,11 @@ public void visit(SQLite3ResultMap tableSummary) { sb.append(" = "); sb.append(vs.get(columnNames.get(j)).get(i).toString()); } - sb.append(" AND "); + if (j < columnNames.size() - 1) { + sb.append(" AND "); + } } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.append("THEN "); + sb.append(" THEN "); visit(summary.get(i)); sb.append(" "); } diff --git a/src/sqlancer/sqlite3/ast/SQLite3Expression.java b/src/sqlancer/sqlite3/ast/SQLite3Expression.java index f3542d863..f8070d56a 100644 --- a/src/sqlancer/sqlite3/ast/SQLite3Expression.java +++ b/src/sqlancer/sqlite3/ast/SQLite3Expression.java @@ -1631,12 +1631,14 @@ public String getString() { StringBuilder sb = new StringBuilder(); sb.append(table.getName()); sb.append("("); + Boolean isFirstColumn = true; for (SQLite3Column c : this.table.getColumns()) { + if (!isFirstColumn) { + sb.append(", "); + } sb.append(c.getName()); - sb.append(", "); + isFirstColumn = false; } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); sb.append(")"); return sb.toString(); } diff --git a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java index 17444026a..b302bdd56 100644 --- a/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java +++ b/src/sqlancer/sqlite3/oracle/SQLite3CODDTestOracle.java @@ -781,10 +781,11 @@ private SQLite3Table createTemporaryTable(SQLite3Select select, String tableName columnTypeName = ""; } } - sb.append("c" + String.valueOf(i) + " " + columnTypeName + ", "); + sb.append("c" + String.valueOf(i) + " " + columnTypeName); + if (i < columnNumber - 1) { + sb.append(", "); + } } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); sb.append(");"); String crateTableString = sb.toString(); if (options.logEachSelect()) { diff --git a/src/sqlancer/tidb/oracle/TiDBCODDTestOracle.java b/src/sqlancer/tidb/oracle/TiDBCODDTestOracle.java index 7b5e6abde..d9199e9ee 100644 --- a/src/sqlancer/tidb/oracle/TiDBCODDTestOracle.java +++ b/src/sqlancer/tidb/oracle/TiDBCODDTestOracle.java @@ -688,10 +688,11 @@ private TiDBTable createTemporaryTable(TiDBSelect select, String tableName, Stri if (idxTypeMap.get(i) != null) { columnTypeName = idxTypeMap.get(i).getPrimitiveDataType().name(); } - sb.append("c" + String.valueOf(i) + " " + columnTypeName + ", "); + sb.append("c" + String.valueOf(i) + " " + columnTypeName); + if (i < columnNumber - 1) { + sb.append(", "); + } } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); sb.append(");"); String crateTableString = sb.toString(); if (options.logEachSelect()) { diff --git a/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java b/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java index a6c12a327..f4c17bfac 100644 --- a/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java +++ b/src/sqlancer/tidb/visitor/TiDBToStringVisitor.java @@ -264,16 +264,19 @@ public void visit(TiDBValues values) { // sb.append("("); for (int i = 0; i < size; i++) { sb.append("("); + Boolean isFirstColumn = true; for (TiDBColumn name : vs.keySet()) { + if (!isFirstColumn) { + sb.append(", "); + } visit(vs.get(name).get(i)); + isFirstColumn = false; + } + sb.append(")"); + if (i < size - 1) { sb.append(", "); } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.append("), "); } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); } @Override @@ -284,16 +287,19 @@ public void visit(TiDBValuesRow values) { sb.append("(VALUES "); for (int i = 0; i < size; i++) { sb.append("ROW("); + Boolean isFirstColumn = true; for (TiDBColumn name : vs.keySet()) { + if (!isFirstColumn) { + sb.append(", "); + } visit(vs.get(name).get(i)); + isFirstColumn = false; + } + sb.append(")"); + if (i < size - 1) { sb.append(", "); } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.append("), "); } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); sb.append(")"); } @@ -315,15 +321,15 @@ public void visit(TiDBResultMap tableSummary) { int size = vs.get(vs.keySet().iterator().next()).size(); if (size == 0) { sb.append("("); + Boolean isFirstColumn = true; for (TiDBColumnReference tr: vs.keySet()) { + if (!isFirstColumn) { + sb.append(" AND "); + } visit(tr); - sb.append(" IS NULL AND "); + sb.append(" IS NULL"); + isFirstColumn = false; } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); sb.append(")"); return; } @@ -331,24 +337,21 @@ public void visit(TiDBResultMap tableSummary) { sb.append(" CASE "); for (int i = 0; i < size; i++) { sb.append("WHEN "); + Boolean isFirstColumn = true; for (TiDBColumnReference tr: vs.keySet()) { + if (!isFirstColumn) { + sb.append(" AND "); + } visit(tr); if (vs.get(tr).get(i) instanceof TiDBNullConstant) { sb.append(" IS NULL"); } else { sb.append(" = "); sb.append(vs.get(tr).get(i).toString()); - // if (values.getColumns().get(j).getType() != null) { - // sb.append("::" + values.getColumns().get(j).getType().toString()); - // } } - sb.append(" AND "); + isFirstColumn = false; } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); - sb.append("THEN "); + sb.append(" THEN "); visit(results.get(i)); // sb.append("::" + summaryDataType.toString() + " "); sb.append(" "); @@ -373,13 +376,14 @@ public void visit(TiDBAllOperator allOperation) { } else { sb.append(" SELECT "); } - + Boolean isFirstColumn = true; for (TiDBColumn name : vs.keySet()) { + if (!isFirstColumn) { + sb.append(", "); + } visit(vs.get(name).get(i)); - sb.append(", "); + isFirstColumn = false; } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); } } else { visit(allOperation.getRightExpr()); @@ -404,13 +408,14 @@ public void visit(TiDBAnyOperator anyOperation) { } else { sb.append(" SELECT "); } - + Boolean isFirstColumn = true; for (TiDBColumn name : vs.keySet()) { + if (!isFirstColumn) { + sb.append(", "); + } visit(vs.get(name).get(i)); - sb.append(", "); + isFirstColumn = false; } - sb.deleteCharAt(sb.length() - 1); - sb.deleteCharAt(sb.length() - 1); } } else { visit(anyOperation.getRightExpr()); From 9581dbbcb7b66594e0d8f1abfc8d1285deebe9c7 Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Wed, 12 Mar 2025 10:14:39 +0800 Subject: [PATCH 15/16] modify DuckDBConstantWithType to DuckDBTypeCast --- .../duckdb/DuckDBToStringVisitor.java | 13 ++------ .../duckdb/ast/DuckDBConstantWithType.java | 31 ------------------- .../duckdb/test/DuckDBCODDTestOracle.java | 15 +++++---- 3 files changed, 10 insertions(+), 49 deletions(-) delete mode 100644 src/sqlancer/duckdb/ast/DuckDBConstantWithType.java diff --git a/src/sqlancer/duckdb/DuckDBToStringVisitor.java b/src/sqlancer/duckdb/DuckDBToStringVisitor.java index c1e1af8b5..c4bbf367d 100644 --- a/src/sqlancer/duckdb/DuckDBToStringVisitor.java +++ b/src/sqlancer/duckdb/DuckDBToStringVisitor.java @@ -7,7 +7,6 @@ import sqlancer.common.ast.newast.TableReferenceNode; import sqlancer.duckdb.ast.DuckDBColumnReference; import sqlancer.duckdb.ast.DuckDBConstant; -import sqlancer.duckdb.ast.DuckDBConstantWithType; import sqlancer.duckdb.ast.DuckDBExpression; import sqlancer.duckdb.ast.DuckDBExpressionBag; import sqlancer.duckdb.ast.DuckDBJoin; @@ -27,8 +26,6 @@ public void visitSpecific(DuckDBExpression expr) { visit((DuckDBSelect) expr); } else if (expr instanceof DuckDBJoin) { visit((DuckDBJoin) expr); - } else if (expr instanceof DuckDBConstantWithType) { - visit((DuckDBConstantWithType) expr); } else if (expr instanceof DuckDBResultMap) { visit((DuckDBResultMap) expr); } else if (expr instanceof DuckDBTypeCast) { @@ -107,10 +104,6 @@ private void visit(DuckDBSelect select) { } } - public void visit(DuckDBConstantWithType expr) { - sb.append(expr.toString()); - } - public void visit(DuckDBResultMap expr) { // use CASE WHEN to express the constant result of a expression LinkedHashMap> dbstate = expr.getDbStates(); @@ -131,9 +124,9 @@ public void visit(DuckDBResultMap expr) { visit(columnRef); if (dbstate.get(columnRef).get(i) instanceof DuckDBNullConstant) { sb.append(" IS NULL"); - } else if (dbstate.get(columnRef).get(i) instanceof DuckDBConstantWithType) { - DuckDBConstantWithType ct = (DuckDBConstantWithType) dbstate.get(columnRef).get(i); - if (ct.getConstant() instanceof DuckDBNullConstant) { + } else if (dbstate.get(columnRef).get(i) instanceof DuckDBTypeCast) { + DuckDBTypeCast ct = (DuckDBTypeCast) dbstate.get(columnRef).get(i); + if (ct.getExpression() instanceof DuckDBNullConstant) { sb.append(" IS NULL"); } else { sb.append(" = "); diff --git a/src/sqlancer/duckdb/ast/DuckDBConstantWithType.java b/src/sqlancer/duckdb/ast/DuckDBConstantWithType.java deleted file mode 100644 index f6c43cc3f..000000000 --- a/src/sqlancer/duckdb/ast/DuckDBConstantWithType.java +++ /dev/null @@ -1,31 +0,0 @@ -package sqlancer.duckdb.ast; - -import sqlancer.duckdb.DuckDBSchema.DuckDBCompositeDataType; -import sqlancer.duckdb.ast.DuckDBConstant.DuckDBNullConstant; - -public class DuckDBConstantWithType implements DuckDBExpression { - private DuckDBConstant constant; - private DuckDBCompositeDataType type; - - public DuckDBConstantWithType(DuckDBConstant constant, DuckDBCompositeDataType type) { - this.constant = constant; - this.type = type; - } - - public String toString() { - if (constant instanceof DuckDBNullConstant) { - return constant.toString(); - } - else { - return "(" + constant.toString() + ")::" + type.toString(); - } - } - - public DuckDBConstant getConstant() { - return constant; - } - - public DuckDBCompositeDataType getType() { - return type; - } -} diff --git a/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java b/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java index 0e835e47c..90918259e 100644 --- a/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java +++ b/src/sqlancer/duckdb/test/DuckDBCODDTestOracle.java @@ -35,7 +35,6 @@ import sqlancer.duckdb.ast.DuckDBBinaryOperator; import sqlancer.duckdb.ast.DuckDBColumnReference; import sqlancer.duckdb.ast.DuckDBConstant; -import sqlancer.duckdb.ast.DuckDBConstantWithType; import sqlancer.duckdb.ast.DuckDBExistsOperator; import sqlancer.duckdb.ast.DuckDBExpression; import sqlancer.duckdb.ast.DuckDBExpressionBag; @@ -166,7 +165,7 @@ else if (auxiliaryQueryResult.size() == 1 && auxiliaryQueryResult.get(auxiliaryQ // folded query DuckDBCompositeDataType constantType = this.getColumnTypeFromSelect(auxiliaryQuery).get(0); DuckDBConstant constant = (DuckDBConstant) auxiliaryQueryResult.get(auxiliaryQueryResult.keySet().toArray()[0]).get(0); - DuckDBConstantWithType equivalentExpr = new DuckDBConstantWithType(constant, constantType); + DuckDBTypeCast equivalentExpr = new DuckDBTypeCast(constant, constantType); specificCondition.updateInnerExpr(equivalentExpr); foldedQueryString = DuckDBToStringVisitor.asString(originalQuery); foldedResult = getQueryResult(foldedQueryString, state); @@ -201,7 +200,7 @@ else if (auxiliaryQueryResult.size() == 1 && Randomly.getBoolean()) { List rowRs = new ArrayList<>(); for (int j = 0; j < typeList.size(); ++j) { DuckDBConstant c = (DuckDBConstant) auxiliaryQueryResult.get("c" + String.valueOf(j)).get(i); - rowRs.add(new DuckDBConstantWithType(c, typeList.get(j))); + rowRs.add(new DuckDBTypeCast(c, typeList.get(j))); } DuckDBValues valueRow = new DuckDBValues(rowRs); values.add(valueRow); @@ -234,7 +233,7 @@ else if (auxiliaryQueryResult.size() == 1 && Randomly.getBoolean()) { List rowRs = new ArrayList<>(); for (int j = 0; j < typeList.size(); ++j) { DuckDBConstant c = (DuckDBConstant) auxiliaryQueryResult.get("c" + String.valueOf(j)).get(i); - rowRs.add(new DuckDBConstantWithType(c, typeList.get(j))); + rowRs.add(new DuckDBTypeCast(c, typeList.get(j))); } DuckDBValues valueRow = new DuckDBValues(rowRs); values.add(valueRow); @@ -490,7 +489,7 @@ private DuckDBSelect genSelectWithCorrelatedSubquery() { List constantRes = new ArrayList<>(); for (DuckDBExpression e : summary) { DuckDBConstant c = (DuckDBConstant) e; - constantRes.add(new DuckDBConstantWithType(c, exprType)); + constantRes.add(new DuckDBTypeCast(c, exprType)); } LinkedHashMap> dbstate = new LinkedHashMap<>(); @@ -506,7 +505,7 @@ private DuckDBSelect genSelectWithCorrelatedSubquery() { if (columnsType.get(i).toString().equals("REAL") || columnsType.get(i).toString().startsWith("FLOAT")) { throw new IgnoreMeException(); } - constants.add(new DuckDBConstantWithType(c, columnsType.get(i))); + constants.add(new DuckDBTypeCast(c, columnsType.get(i))); } dbstate.put(cRef, constants); } @@ -584,7 +583,7 @@ private DuckDBSelect genSimpleSelect() { List constantRes = new ArrayList<>(); for (DuckDBExpression e : summary) { DuckDBConstant c = (DuckDBConstant) e; - constantRes.add(new DuckDBConstantWithType(c, exprType)); + constantRes.add(new DuckDBTypeCast(c, exprType)); } LinkedHashMap> dbstate = new LinkedHashMap<>(); @@ -600,7 +599,7 @@ private DuckDBSelect genSimpleSelect() { if (columnsType.get(i).toString().equals("REAL") || columnsType.get(i).toString().startsWith("FLOAT")) { throw new IgnoreMeException(); } - constants.add(new DuckDBConstantWithType(c, columnsType.get(i))); + constants.add(new DuckDBTypeCast(c, columnsType.get(i))); } dbstate.put(cRef, constants); } From f99cc40ecd6838ae11fb84792d6992acaa55badf Mon Sep 17 00:00:00 2001 From: DerZc <798604270@qq.com> Date: Wed, 12 Mar 2025 10:34:15 +0800 Subject: [PATCH 16/16] remove an internel error from the ignore list --- src/sqlancer/duckdb/DuckDBErrors.java | 1 - 1 file changed, 1 deletion(-) diff --git a/src/sqlancer/duckdb/DuckDBErrors.java b/src/sqlancer/duckdb/DuckDBErrors.java index d8ca8789b..3b9b3aea0 100644 --- a/src/sqlancer/duckdb/DuckDBErrors.java +++ b/src/sqlancer/duckdb/DuckDBErrors.java @@ -67,7 +67,6 @@ public static List getExpressionErrors() { errors.add("must appear in the GROUP BY clause or be used in an aggregate function"); errors.add("must appear in the GROUP BY clause or must be part of an aggregate function"); errors.add("GROUP BY term out of range - should be between"); - errors.add("INTERNAL Error: Failed to bind column reference"); errors.add("Binder Error: Aggregate with only constant parameters has to be bound in the root subquery"); errors.add("COLLATE can only be applied to varchar columns");