Skip to content

Commit 354d591

Browse files
ZuoTiJiaZuoTiJia
authored
Support CnosDB (#697)
* Add Cnosdb (#1) Add Cnosdb * cnosdb: fix check style error * cnosdb: fix compiler error * cnosdb: remove unused * cnosdb: remove some pqs impl * cnosdb: remove dead code * add query execute count * refactor expectedError * cnosdb: remove unsuccessful count
1 parent 39c48cd commit 354d591

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

53 files changed

+4278
-0
lines changed

‎pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -358,6 +358,11 @@
358358
<version>2.7.1</version>
359359
<scope>runtime</scope>
360360
</dependency>
361+
<dependency>
362+
<groupId>org.apache.commons</groupId>
363+
<artifactId>commons-csv</artifactId>
364+
<version>1.9.0</version>
365+
</dependency>
361366
</dependencies>
362367
<reporting>
363368
<plugins>

‎src/check_names.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,3 +26,4 @@ def verify_prefix(prefix, files):
2626
verify_prefix('Postgres', get_java_files("sqlancer/postgres/"))
2727
verify_prefix('SQLite3', get_java_files("sqlancer/sqlite3/"))
2828
verify_prefix('TiDB', get_java_files("sqlancer/tidb/"))
29+
verify_prefix('CnosDB', get_java_files("sqlancer/cnosdb/"))

‎src/sqlancer/cnosdb/CnosDBBugs.java‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
package sqlancer.cnosdb;
2+
3+
public final class CnosDBBugs {
4+
5+
// https://github.com/cnosdb/cnosdb/issues/786
6+
public static final boolean BUG786 = true;
7+
8+
// https://github.com/apache/arrow-rs/issues/3547
9+
public static final boolean BUG3547 = true;
10+
11+
private CnosDBBugs() {
12+
}
13+
}

‎src/sqlancer/cnosdb/CnosDBComparatorHelper.java‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
package sqlancer.cnosdb;
2+
3+
import java.io.IOException;
4+
import java.util.ArrayList;
5+
import java.util.HashSet;
6+
import java.util.List;
7+
import java.util.Set;
8+
import java.util.function.UnaryOperator;
9+
import java.util.stream.Collectors;
10+
11+
import sqlancer.IgnoreMeException;
12+
import sqlancer.cnosdb.client.CnosDBResultSet;
13+
import sqlancer.cnosdb.query.CnosDBSelectQuery;
14+
import sqlancer.common.query.ExpectedErrors;
15+
16+
public final class CnosDBComparatorHelper {
17+
18+
private CnosDBComparatorHelper() {
19+
}
20+
21+
public static List<String> getResultSetFirstColumnAsString(String queryString, ExpectedErrors errors,
22+
CnosDBGlobalState state) throws Exception {
23+
if (state.getOptions().logEachSelect()) {
24+
// TODO: refactor me
25+
state.getLogger().writeCurrent(queryString);
26+
try {
27+
state.getLogger().getCurrentFileWriter().flush();
28+
} catch (IOException e) {
29+
// TODO Auto-generated catch block
30+
e.printStackTrace();
31+
}
32+
}
33+
CnosDBSelectQuery q = new CnosDBSelectQuery(queryString, errors);
34+
List<String> result = new ArrayList<>();
35+
CnosDBResultSet resultSet;
36+
try {
37+
q.executeAndGet(state);
38+
resultSet = q.getResultSet();
39+
if (resultSet == null) {
40+
throw new AssertionError(q);
41+
}
42+
while (resultSet.next()) {
43+
result.add(resultSet.getString(1));
44+
}
45+
} catch (Exception e) {
46+
if (e instanceof IgnoreMeException) {
47+
throw e;
48+
}
49+
if (e instanceof NumberFormatException) {
50+
throw new IgnoreMeException();
51+
}
52+
if (e.getMessage() == null) {
53+
throw new AssertionError(queryString, e);
54+
}
55+
if (errors.errorIsExpected(e.getMessage())) {
56+
throw new IgnoreMeException();
57+
}
58+
throw new AssertionError(queryString, e);
59+
}
60+
61+
return result;
62+
}
63+
64+
public static void assumeResultSetsAreEqual(List<String> resultSet, List<String> secondResultSet,
65+
String originalQueryString, List<String> combinedString, CnosDBGlobalState state) {
66+
if (resultSet.size() != secondResultSet.size()) {
67+
String queryFormatString = "-- %s;\n-- cardinality: %d";
68+
String firstQueryString = String.format(queryFormatString, originalQueryString, resultSet.size());
69+
String secondQueryString = String.format(queryFormatString, String.join(";", combinedString),
70+
secondResultSet.size());
71+
state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString));
72+
String assertionMessage = String.format("the size of the result sets mismatch (%d and %d)!\n%s\n%s",
73+
resultSet.size(), secondResultSet.size(), firstQueryString, secondQueryString);
74+
throw new AssertionError(assertionMessage);
75+
}
76+
77+
Set<String> firstHashSet = new HashSet<>(resultSet);
78+
Set<String> secondHashSet = new HashSet<>(secondResultSet);
79+
80+
if (!firstHashSet.equals(secondHashSet)) {
81+
Set<String> firstResultSetMisses = new HashSet<>(firstHashSet);
82+
firstResultSetMisses.removeAll(secondHashSet);
83+
Set<String> secondResultSetMisses = new HashSet<>(secondHashSet);
84+
secondResultSetMisses.removeAll(firstHashSet);
85+
String queryFormatString = "-- %s;\n-- misses: %s";
86+
String firstQueryString = String.format(queryFormatString, originalQueryString, firstResultSetMisses);
87+
String secondQueryString = String.format(queryFormatString, String.join(";", combinedString),
88+
secondResultSetMisses);
89+
// update the SELECT queries to be logged at the bottom of the error log file
90+
state.getState().getLocalState().log(String.format("%s\n%s", firstQueryString, secondQueryString));
91+
String assertionMessage = String.format("the content of the result sets mismatch!\n%s\n%s",
92+
firstQueryString, secondQueryString);
93+
throw new AssertionError(assertionMessage);
94+
}
95+
}
96+
97+
public static void assumeResultSetsAreEqual(List<String> resultSet, List<String> secondResultSet,
98+
String originalQueryString, List<String> combinedString, CnosDBGlobalState state,
99+
UnaryOperator<String> canonicalizationRule) {
100+
// Overloaded version of assumeResultSetsAreEqual that takes a canonicalization function which is applied to
101+
// both result sets before their comparison.
102+
List<String> canonicalizedResultSet = resultSet.stream().map(canonicalizationRule).collect(Collectors.toList());
103+
List<String> canonicalizedSecondResultSet = secondResultSet.stream().map(canonicalizationRule)
104+
.collect(Collectors.toList());
105+
assumeResultSetsAreEqual(canonicalizedResultSet, canonicalizedSecondResultSet, originalQueryString,
106+
combinedString, state);
107+
}
108+
109+
public static List<String> getCombinedResultSet(String firstQueryString, String secondQueryString,
110+
String thirdQueryString, List<String> combinedString, boolean asUnion, CnosDBGlobalState state,
111+
ExpectedErrors errors) throws Exception {
112+
List<String> secondResultSet;
113+
if (asUnion) {
114+
String unionString = firstQueryString + " UNION ALL " + secondQueryString + " UNION ALL "
115+
+ thirdQueryString;
116+
combinedString.add(unionString);
117+
secondResultSet = getResultSetFirstColumnAsString(unionString, errors, state);
118+
} else {
119+
secondResultSet = new ArrayList<>();
120+
secondResultSet.addAll(getResultSetFirstColumnAsString(firstQueryString, errors, state));
121+
secondResultSet.addAll(getResultSetFirstColumnAsString(secondQueryString, errors, state));
122+
secondResultSet.addAll(getResultSetFirstColumnAsString(thirdQueryString, errors, state));
123+
combinedString.add(firstQueryString);
124+
combinedString.add(secondQueryString);
125+
combinedString.add(thirdQueryString);
126+
}
127+
return secondResultSet;
128+
}
129+
130+
public static List<String> getCombinedResultSetNoDuplicates(String firstQueryString, String secondQueryString,
131+
String thirdQueryString, List<String> combinedString, boolean asUnion, CnosDBGlobalState state,
132+
ExpectedErrors errors) throws Exception {
133+
String unionString;
134+
if (asUnion) {
135+
unionString = firstQueryString + " UNION " + secondQueryString + " UNION " + thirdQueryString;
136+
} else {
137+
unionString = "SELECT DISTINCT * FROM (" + firstQueryString + " UNION ALL " + secondQueryString
138+
+ " UNION ALL " + thirdQueryString + ")";
139+
}
140+
List<String> secondResultSet;
141+
combinedString.add(unionString);
142+
secondResultSet = getResultSetFirstColumnAsString(unionString, errors, state);
143+
return secondResultSet;
144+
}
145+
}

‎src/sqlancer/cnosdb/CnosDBCompoundDataType.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
package sqlancer.cnosdb;
2+
3+
import sqlancer.cnosdb.CnosDBSchema.CnosDBDataType;
4+
5+
public final class CnosDBCompoundDataType {
6+
7+
private final CnosDBDataType dataType;
8+
9+
private CnosDBCompoundDataType(CnosDBDataType dataType) {
10+
this.dataType = dataType;
11+
}
12+
13+
public static CnosDBCompoundDataType create(CnosDBDataType type) {
14+
return new CnosDBCompoundDataType(type);
15+
}
16+
17+
public CnosDBDataType getDataType() {
18+
return dataType;
19+
}
20+
}

‎src/sqlancer/cnosdb/CnosDBExpectedError.java‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
package sqlancer.cnosdb;
2+
3+
import java.util.ArrayList;
4+
import java.util.List;
5+
6+
import sqlancer.common.query.ExpectedErrors;
7+
8+
public final class CnosDBExpectedError {
9+
private static final List<String> ERRORS = new ArrayList<>();
10+
11+
private CnosDBExpectedError() {
12+
}
13+
14+
static {
15+
ERRORS.add("have the same name. Consider aliasing");
16+
ERRORS.add(
17+
"error: Optimizer rule 'projection_push_down' failed due to unexpected error: Schema error: Schema contains duplicate qualified field name");
18+
ERRORS.add("Projection references non-aggregate values:");
19+
ERRORS.add("External err: Schema error: No field named");
20+
ERRORS.add(
21+
"Optimizer rule 'common_sub_expression_eliminate' failed due to unexpected error: Schema error: No field named");
22+
ERRORS.add("Binary");
23+
ERRORS.add("Invalid pattern in LIKE expression");
24+
ERRORS.add("If the projection contains the time column, it must contain the field column.");
25+
ERRORS.add("Schema error: No field named");
26+
ERRORS.add("Optimizer rule 'simplify_expressions' failed due to unexpected error:");
27+
ERRORS.add("err: Internal error: Optimizer rule 'projection_push_down' failed due to unexpected error");
28+
ERRORS.add("Schema error: No field named ");
29+
ERRORS.add("err: External err: Schema error: No field named");
30+
ERRORS.add("Optimizer rule 'simplify_expressions' failed due to unexpected error");
31+
ERRORS.add("Csv error: CSV Writer does not support List");
32+
ERRORS.add("This feature is not implemented: cross join.");
33+
ERRORS.add("Execution error: field position must be greater than zero");
34+
ERRORS.add("First argument of `DATE_PART` must be non-null scalar Utf8");
35+
ERRORS.add("Cannot create filter with non-boolean predicate 'NULL' returning Null");
36+
ERRORS.add("requested character too large for encoding.");
37+
ERRORS.add("Can not find compatible types to compare Boolean with [Utf8].");
38+
ERRORS.add("Cannot create filter with non-boolean predicate 'APPROXDISTINCT");
39+
ERRORS.add("HAVING clause references non-aggregate values:");
40+
ERRORS.add("Cannot create filter with non-boolean predicate");
41+
ERRORS.add("negative substring length not allowed");
42+
ERRORS.add("The function Sum does not support inputs of type Boolean.");
43+
ERRORS.add("The function Avg does not support inputs of type Boolean.");
44+
ERRORS.add("Percentile value must be between 0.0 and 1.0 inclusive");
45+
ERRORS.add("Date part '' not supported");
46+
ERRORS.add("Min/Max accumulator not implemented for type Boolean.");
47+
ERRORS.add("meta need get_series_id_by_filter");
48+
ERRORS.add("Arrow: Cast error:");
49+
ERRORS.add("Arrow error: Cast error:");
50+
ERRORS.add("Datafusion: Execution error: Arrow error: External error: Arrow error: Cast error:");
51+
ERRORS.add("Arrow error: Divide by zero error");
52+
ERRORS.add("desired percentile argument must be float literal");
53+
ERRORS.add("Unsupported CAST from Int32 to Timestamp(Nanosecond, None)");
54+
ERRORS.add("Execution error: Date part");
55+
ERRORS.add("Physical plan does not support logical expression MIN(Boolean");
56+
ERRORS.add("The percentile argument for ApproxPercentileCont must be Float64, not Int64");
57+
ERRORS.add("The percentile argument for ApproxPercentileContWithWeight must be Float64, not Int64.");
58+
ERRORS.add("Data type UInt64 not supported for binary operation '#' on dyn arrays.");
59+
ERRORS.add("Arrow: Divide by zero error");
60+
ERRORS.add("The function ApproxPercentileCont does not support inputs of type Null.");
61+
ERRORS.add("can't be evaluated because there isn't a common type to coerce the types to");
62+
ERRORS.add("This was likely caused by a bug in DataFusion's code and we would welcome that you file an bug");
63+
ERRORS.add("The function ApproxMedian does not support inputs of type Null.");
64+
ERRORS.add("null character not permitted.");
65+
ERRORS.add("The percentile argument for ApproxPercentileCont must be Float64, not Null.");
66+
ERRORS.add("This feature is not implemented");
67+
ERRORS.add("The function Avg does not support inputs of type Null.");
68+
ERRORS.add("Coercion from [Utf8, Timestamp(Nanosecond, Some(\\\"+00:00\\\"))]");
69+
ERRORS.add(
70+
"Coercion from [Utf8, Float64, Utf8] to the signature OneOf([Exact([Utf8, Int64]), Exact([LargeUtf8, Int64]), Exact([Utf8, Int64, Utf8]), Exact([LargeUtf8, Int64, Utf8]), Exact([Utf8, Int64, LargeUtf8]), Exact([LargeUtf8, Int64, LargeUtf8])]) failed.");
71+
ERRORS.add("Coercion from");
72+
}
73+
74+
public static ExpectedErrors expectedErrors() {
75+
ExpectedErrors res = new ExpectedErrors();
76+
res.addAll(ERRORS);
77+
return res;
78+
}
79+
80+
}

‎src/sqlancer/cnosdb/CnosDBGlobalState.java‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
package sqlancer.cnosdb;
2+
3+
import sqlancer.ExecutionTimer;
4+
import sqlancer.GlobalState;
5+
import sqlancer.cnosdb.client.CnosDBConnection;
6+
import sqlancer.common.query.Query;
7+
8+
public class CnosDBGlobalState extends GlobalState<CnosDBOptions, CnosDBSchema, CnosDBConnection> {
9+
10+
@Override
11+
protected void executeEpilogue(Query<?> q, boolean success, ExecutionTimer timer) throws Exception {
12+
boolean logExecutionTime = getOptions().logExecutionTime();
13+
if (success && getOptions().printSucceedingStatements()) {
14+
System.out.println(q.getQueryString());
15+
}
16+
if (logExecutionTime) {
17+
getLogger().writeCurrent(" -- " + timer.end().asString());
18+
}
19+
if (q.couldAffectSchema()) {
20+
updateSchema();
21+
}
22+
}
23+
24+
@Override
25+
public CnosDBSchema readSchema() throws Exception {
26+
return CnosDBSchema.fromConnection(getConnection());
27+
}
28+
}

‎src/sqlancer/cnosdb/CnosDBLoggableFactory.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
package sqlancer.cnosdb;
2+
3+
import java.io.PrintWriter;
4+
import java.io.StringWriter;
5+
6+
import sqlancer.cnosdb.query.CnosDBOtherQuery;
7+
import sqlancer.cnosdb.query.CnosDBQueryAdapter;
8+
import sqlancer.common.log.Loggable;
9+
import sqlancer.common.log.LoggableFactory;
10+
import sqlancer.common.log.LoggedString;
11+
import sqlancer.common.query.ExpectedErrors;
12+
import sqlancer.common.query.Query;
13+
14+
public class CnosDBLoggableFactory extends LoggableFactory {
15+
16+
@Override
17+
protected Loggable createLoggable(String input, String suffix) {
18+
String completeString = input;
19+
if (!input.endsWith(";")) {
20+
completeString += ";";
21+
}
22+
if (suffix != null && suffix.length() != 0) {
23+
completeString += suffix;
24+
}
25+
return new LoggedString(completeString);
26+
}
27+
28+
@Override
29+
public CnosDBQueryAdapter getQueryForStateToReproduce(String queryString) {
30+
return new CnosDBOtherQuery(queryString, CnosDBExpectedError.expectedErrors());
31+
}
32+
33+
@Override
34+
public CnosDBQueryAdapter commentOutQuery(Query<?> query) {
35+
String queryString = query.getLogString();
36+
String newQueryString = "-- " + queryString;
37+
ExpectedErrors errors = new ExpectedErrors();
38+
return new CnosDBOtherQuery(newQueryString, errors);
39+
}
40+
41+
@Override
42+
protected Loggable infoToLoggable(String time, String databaseName, String databaseVersion, long seedValue) {
43+
String sb = "-- Time: " + time + "\n" + "-- Database: " + databaseName + "\n" + "-- Database version: "
44+
+ databaseVersion + "\n" + "-- seed value: " + seedValue + "\n";
45+
return new LoggedString(sb);
46+
}
47+
48+
@Override
49+
public Loggable convertStacktraceToLoggable(Throwable throwable) {
50+
StringWriter sw = new StringWriter();
51+
PrintWriter pw = new PrintWriter(sw);
52+
throwable.printStackTrace(pw);
53+
return new LoggedString("--" + sw.toString().replace("\n", "\n--"));
54+
}
55+
}

0 commit comments

Comments
 (0)