Skip to content

Commit 4faa1c9

Browse files
liyxirisliyxiris
committed
[Hive] Support TLP oracle
1 parent 0cae179 commit 4faa1c9

32 files changed

Lines changed: 1585 additions & 3 deletions

‎.github/workflows/main.yml‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -297,6 +297,42 @@ jobs:
297297
- name: Run Tests
298298
run: mvn -Dtest=TestH2 test
299299

300+
hive:
301+
name: DBMS Tests (Hive)
302+
runs-on: ubuntu-latest
303+
services:
304+
metastore:
305+
image: apache/hive:4.0.1
306+
env:
307+
SERVICE_NAME: 'metastore'
308+
ports:
309+
- 9083:9083
310+
volumes:
311+
- warehouse:/opt/hive/data/warehouse
312+
hiveserver2:
313+
image: apache/hive:4.0.1
314+
env:
315+
SERVICE_NAME: 'hiveserver2'
316+
ports:
317+
- 10000:10000
318+
- 10002:10002
319+
volumes:
320+
- warehouse:/opt/hive/data/warehouse
321+
steps:
322+
- uses: actions/checkout@v3
323+
with:
324+
fetch-depth: 0
325+
- name: Set up JDK 11
326+
uses: actions/setup-java@v3
327+
with:
328+
distribution: 'temurin'
329+
java-version: '11'
330+
cache: 'maven'
331+
- name: Build SQLancer
332+
run: mvn -B package -DskipTests=true
333+
- name: Run Tests
334+
run: HIVE_AVAILABLE=true mvn -Dtest=TestHiveTLP test
335+
300336
hsqldb:
301337
name: DBMS Tests (HSQLB)
302338
runs-on: ubuntu-latest

‎.gitignore‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,6 @@
11
target/
22
.classpath
3-
.settings/org.eclipse.core.resources.prefs
4-
.settings/org.eclipse.m2e.core.prefs
5-
.settings/org.eclipse.jdt.core.prefs
3+
.settings/
64
.vscode
75
.project
86
.checkstyle

‎pom.xml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -377,6 +377,16 @@
377377
<groupId>org.apache.arrow</groupId>
378378
<artifactId>flight-sql-jdbc-driver</artifactId>
379379
<version>16.1.0</version>
380+
</dependency>
381+
<dependency>
382+
<groupId>org.apache.hive</groupId>
383+
<artifactId>hive-jdbc</artifactId>
384+
<version>3.1.3</version>
385+
</dependency>
386+
<dependency>
387+
<groupId>org.apache.hive</groupId>
388+
<artifactId>hive-serde</artifactId>
389+
<version>3.1.3</version>
380390
</dependency>
381391
</dependencies>
382392
<reporting>

‎src/sqlancer/Main.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import sqlancer.doris.DorisProvider;
3535
import sqlancer.duckdb.DuckDBProvider;
3636
import sqlancer.h2.H2Provider;
37+
import sqlancer.hive.HiveProvider;
3738
import sqlancer.hsqldb.HSQLDBProvider;
3839
import sqlancer.mariadb.MariaDBProvider;
3940
import sqlancer.materialize.MaterializeProvider;
@@ -735,6 +736,7 @@ private static void checkForIssue799(List<DatabaseProvider<?, ?, ?>> providers)
735736
providers.add(new DorisProvider());
736737
providers.add(new DuckDBProvider());
737738
providers.add(new H2Provider());
739+
providers.add(new HiveProvider());
738740
providers.add(new HSQLDBProvider());
739741
providers.add(new MariaDBProvider());
740742
providers.add(new MaterializeProvider());

‎src/sqlancer/hive/HiveErrors.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
package sqlancer.hive;
2+
3+
import java.util.ArrayList;
4+
import java.util.List;
5+
6+
import sqlancer.common.query.ExpectedErrors;
7+
8+
public class HiveErrors {
9+
10+
private HiveErrors() {
11+
}
12+
13+
public static List<String> getExpressionErrors() {
14+
ArrayList<String> errors = new ArrayList<>();
15+
16+
errors.add("cannot recognize input near");
17+
errors.add("Argument type mismatch");
18+
errors.add("Error while compiling statement");
19+
20+
return errors;
21+
}
22+
23+
public static void addExpressionErrors(ExpectedErrors errors) {
24+
errors.addAll(getExpressionErrors());
25+
}
26+
27+
public static List<String> getInsertErrors() {
28+
ArrayList<String> errors = new ArrayList<>();
29+
30+
errors.add("Either CHECK or NOT NULL constraint violated!");
31+
errors.add("Error running query");
32+
errors.add("is different from preceding arguments");
33+
34+
return errors;
35+
}
36+
37+
public static void addInsertErrors(ExpectedErrors errors) {
38+
errors.addAll(getInsertErrors());
39+
}
40+
}

‎src/sqlancer/hive/HiveGlobalState.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
package sqlancer.hive;
2+
3+
import sqlancer.SQLGlobalState;
4+
5+
public class HiveGlobalState extends SQLGlobalState<HiveOptions, HiveSchema> {
6+
7+
@Override
8+
protected HiveSchema readSchema() throws Exception {
9+
return HiveSchema.fromConnection(getConnection(), getDatabaseName());
10+
}
11+
}

‎src/sqlancer/hive/HiveOptions.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
package sqlancer.hive;
2+
3+
import com.beust.jcommander.Parameter;
4+
import com.beust.jcommander.Parameters;
5+
import sqlancer.common.oracle.TestOracle;
6+
import sqlancer.DBMSSpecificOptions;
7+
import sqlancer.hive.oracle.HiveTLPWhereOracle;
8+
import sqlancer.OracleFactory;
9+
10+
import java.util.Arrays;
11+
import java.util.List;
12+
13+
@Parameters(separators = "=", commandDescription = "Hive (default port: " + HiveOptions.DEFAULT_PORT
14+
+ ", default host: " + HiveOptions.DEFAULT_HOST + ")")
15+
public class HiveOptions implements DBMSSpecificOptions<HiveOptions.HiveOracleFactory> {
16+
public static final String DEFAULT_HOST = "localhost";
17+
public static final int DEFAULT_PORT = 10000;
18+
19+
@Parameter(names = "--oracle")
20+
public List<HiveOracleFactory> oracle = Arrays.asList(HiveOracleFactory.TLPWhere);
21+
22+
public enum HiveOracleFactory implements OracleFactory<HiveGlobalState> {
23+
TLPWhere {
24+
@Override
25+
public TestOracle<HiveGlobalState> create(HiveGlobalState globalState) {
26+
return new HiveTLPWhereOracle(globalState);
27+
}
28+
};
29+
}
30+
31+
@Override
32+
public List<HiveOracleFactory> getTestOracleFactory() {
33+
return oracle;
34+
}
35+
}

‎src/sqlancer/hive/HiveProvider.java‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
package sqlancer.hive;
2+
3+
import sqlancer.AbstractAction;
4+
import sqlancer.DatabaseProvider;
5+
import sqlancer.IgnoreMeException;
6+
import sqlancer.MainOptions;
7+
import sqlancer.Randomly;
8+
import sqlancer.SQLConnection;
9+
import sqlancer.SQLProviderAdapter;
10+
import sqlancer.StatementExecutor;
11+
import sqlancer.common.query.SQLQueryAdapter;
12+
import sqlancer.common.query.SQLQueryProvider;
13+
import sqlancer.hive.gen.HiveInsertGenerator;
14+
import sqlancer.hive.gen.HiveTableGenerator;
15+
16+
import java.sql.Connection;
17+
import java.sql.DriverManager;
18+
import java.sql.SQLException;
19+
import java.sql.Statement;
20+
21+
import com.google.auto.service.AutoService;
22+
23+
@AutoService(DatabaseProvider.class)
24+
public class HiveProvider extends SQLProviderAdapter<HiveGlobalState, HiveOptions> {
25+
26+
public HiveProvider() {
27+
super(HiveGlobalState.class, HiveOptions.class);
28+
}
29+
30+
public enum Action implements AbstractAction<HiveGlobalState> {
31+
32+
INSERT(HiveInsertGenerator::getQuery);
33+
34+
private final SQLQueryProvider<HiveGlobalState> sqlQueryProvider;
35+
36+
Action(SQLQueryProvider<HiveGlobalState> sqlQueryProvider) {
37+
this.sqlQueryProvider = sqlQueryProvider;
38+
}
39+
40+
@Override
41+
public SQLQueryAdapter getQuery(HiveGlobalState state) throws Exception {
42+
return sqlQueryProvider.getQuery(state);
43+
}
44+
}
45+
46+
private static int mapActions(HiveGlobalState globalState, Action a) {
47+
Randomly r = globalState.getRandomly();
48+
switch (a) {
49+
case INSERT:
50+
return r.getInteger(0, globalState.getOptions().getMaxNumberInserts());
51+
default:
52+
throw new AssertionError(a);
53+
}
54+
}
55+
56+
@Override
57+
public void generateDatabase(HiveGlobalState globalState) throws Exception {
58+
for (int i = 0; i < Randomly.fromOptions(1, 2); i++) {
59+
boolean success;
60+
do {
61+
String tableName = globalState.getSchema().getFreeTableName();
62+
SQLQueryAdapter qt = HiveTableGenerator.generate(globalState, tableName);
63+
success = globalState.executeStatement(qt);
64+
} while(!success);
65+
}
66+
if (globalState.getSchema().getDatabaseTables().isEmpty()) {
67+
throw new IgnoreMeException(); // TODO
68+
}
69+
70+
StatementExecutor<HiveGlobalState, Action> se = new StatementExecutor<HiveGlobalState, Action>(
71+
globalState, Action.values(),
72+
HiveProvider::mapActions, (q) -> {
73+
if (globalState.getSchema().getDatabaseTables().isEmpty()) {
74+
throw new IgnoreMeException();
75+
}
76+
});
77+
se.executeStatements();
78+
}
79+
80+
@Override
81+
public SQLConnection createDatabase(HiveGlobalState globalState) throws SQLException {
82+
String username = globalState.getOptions().getUserName();
83+
String password = globalState.getOptions().getPassword();
84+
String host = globalState.getOptions().getHost();
85+
int port = globalState.getOptions().getPort();
86+
if (host == null) {
87+
host = HiveOptions.DEFAULT_HOST;
88+
}
89+
if (port == MainOptions.NO_SET_PORT) {
90+
port = HiveOptions.DEFAULT_PORT;
91+
}
92+
93+
String databaseName = globalState.getDatabaseName();
94+
95+
String url = String.format("jdbc:hive2://%s:%d/%s", host, port, "default");
96+
Connection con = DriverManager.getConnection(url, username, password);
97+
globalState.getState().logStatement("DROP DATABASE IF EXISTS " + databaseName + " CASCADE");
98+
globalState.getState().logStatement("CREATE DATABASE " + databaseName);
99+
globalState.getState().logStatement("USE " + databaseName);
100+
try (Statement s = con.createStatement()) {
101+
s.execute("DROP DATABASE IF EXISTS " + databaseName + " CASCADE");
102+
}
103+
try (Statement s = con.createStatement()) {
104+
s.execute("CREATE DATABASE " + databaseName);
105+
}
106+
try (Statement s = con.createStatement()) {
107+
s.execute("USE " + databaseName);
108+
}
109+
con.close();
110+
con = DriverManager.getConnection(
111+
String.format("jdbc:hive2://%s:%d/%s", host, port, databaseName,
112+
username, password));
113+
114+
return new SQLConnection(con);
115+
}
116+
117+
@Override
118+
public String getDBMSName() {
119+
return "hive";
120+
}
121+
}

0 commit comments

Comments
 (0)