Skip to content

Commit a9b93e0

Browse files
hvanhovellrxin
hvanhovell
authored and
rxin
committed
[SPARK-14211][SQL] Remove ANTLR3 based parser
### What changes were proposed in this pull request? This PR removes the ANTLR3 based parser, and moves the new ANTLR4 based parser into the `org.apache.spark.sql.catalyst.parser package`. ### How was this patch tested? Existing unit tests. cc rxin andrewor14 yhuai Author: Herman van Hovell <[email protected]> Closes apache#12071 from hvanhovell/SPARK-14211.
1 parent 26445c2 commit a9b93e0

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

47 files changed

+411
-8283
lines changed

‎dev/deps/spark-deps-hadoop-2.2‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@ JavaEWAH-0.3.2.jar
22
RoaringBitmap-0.5.11.jar
33
ST4-4.0.4.jar
44
activation-1.1.jar
5-
antlr-runtime-3.5.2.jar
5+
antlr-2.7.7.jar
6+
antlr-runtime-3.4.jar
67
antlr4-runtime-4.5.2-1.jar
78
aopalliance-1.0.jar
89
apache-log4j-extras-1.2.17.jar
@@ -173,6 +174,7 @@ spire_2.11-0.7.4.jar
173174
stax-api-1.0-2.jar
174175
stax-api-1.0.1.jar
175176
stream-2.7.0.jar
177+
stringtemplate-3.2.1.jar
176178
super-csv-2.2.0.jar
177179
univocity-parsers-1.5.6.jar
178180
unused-1.0.0.jar

‎dev/deps/spark-deps-hadoop-2.3‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@ JavaEWAH-0.3.2.jar
22
RoaringBitmap-0.5.11.jar
33
ST4-4.0.4.jar
44
activation-1.1.1.jar
5-
antlr-runtime-3.5.2.jar
5+
antlr-2.7.7.jar
6+
antlr-runtime-3.4.jar
67
antlr4-runtime-4.5.2-1.jar
78
aopalliance-1.0.jar
89
apache-log4j-extras-1.2.17.jar
@@ -164,6 +165,7 @@ spire_2.11-0.7.4.jar
164165
stax-api-1.0-2.jar
165166
stax-api-1.0.1.jar
166167
stream-2.7.0.jar
168+
stringtemplate-3.2.1.jar
167169
super-csv-2.2.0.jar
168170
univocity-parsers-1.5.6.jar
169171
unused-1.0.0.jar

‎dev/deps/spark-deps-hadoop-2.4‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@ JavaEWAH-0.3.2.jar
22
RoaringBitmap-0.5.11.jar
33
ST4-4.0.4.jar
44
activation-1.1.1.jar
5-
antlr-runtime-3.5.2.jar
5+
antlr-2.7.7.jar
6+
antlr-runtime-3.4.jar
67
antlr4-runtime-4.5.2-1.jar
78
aopalliance-1.0.jar
89
apache-log4j-extras-1.2.17.jar
@@ -165,6 +166,7 @@ spire_2.11-0.7.4.jar
165166
stax-api-1.0-2.jar
166167
stax-api-1.0.1.jar
167168
stream-2.7.0.jar
169+
stringtemplate-3.2.1.jar
168170
super-csv-2.2.0.jar
169171
univocity-parsers-1.5.6.jar
170172
unused-1.0.0.jar

‎dev/deps/spark-deps-hadoop-2.6‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@ JavaEWAH-0.3.2.jar
22
RoaringBitmap-0.5.11.jar
33
ST4-4.0.4.jar
44
activation-1.1.1.jar
5-
antlr-runtime-3.5.2.jar
5+
antlr-2.7.7.jar
6+
antlr-runtime-3.4.jar
67
antlr4-runtime-4.5.2-1.jar
78
aopalliance-1.0.jar
89
apache-log4j-extras-1.2.17.jar
@@ -171,6 +172,7 @@ spire_2.11-0.7.4.jar
171172
stax-api-1.0-2.jar
172173
stax-api-1.0.1.jar
173174
stream-2.7.0.jar
175+
stringtemplate-3.2.1.jar
174176
super-csv-2.2.0.jar
175177
univocity-parsers-1.5.6.jar
176178
unused-1.0.0.jar

‎dev/deps/spark-deps-hadoop-2.7‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@ JavaEWAH-0.3.2.jar
22
RoaringBitmap-0.5.11.jar
33
ST4-4.0.4.jar
44
activation-1.1.1.jar
5-
antlr-runtime-3.5.2.jar
5+
antlr-2.7.7.jar
6+
antlr-runtime-3.4.jar
67
antlr4-runtime-4.5.2-1.jar
78
aopalliance-1.0.jar
89
apache-log4j-extras-1.2.17.jar
@@ -172,6 +173,7 @@ spire_2.11-0.7.4.jar
172173
stax-api-1.0-2.jar
173174
stax-api-1.0.1.jar
174175
stream-2.7.0.jar
176+
stringtemplate-3.2.1.jar
175177
super-csv-2.2.0.jar
176178
univocity-parsers-1.5.6.jar
177179
unused-1.0.0.jar

‎pom.xml‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -177,7 +177,6 @@
177177
<jodd.version>3.5.2</jodd.version>
178178
<jsr305.version>1.3.9</jsr305.version>
179179
<libthrift.version>0.9.2</libthrift.version>
180-
<antlr.version>3.5.2</antlr.version>
181180
<antlr4.version>4.5.2-1</antlr4.version>
182181

183182
<test.java.home>${java.home}</test.java.home>
@@ -1755,11 +1754,6 @@
17551754
</exclusion>
17561755
</exclusions>
17571756
</dependency>
1758-
<dependency>
1759-
<groupId>org.antlr</groupId>
1760-
<artifactId>antlr-runtime</artifactId>
1761-
<version>${antlr.version}</version>
1762-
</dependency>
17631757
<dependency>
17641758
<groupId>org.antlr</groupId>
17651759
<artifactId>antlr4-runtime</artifactId>

‎project/SparkBuild.scala‎

Lines changed: 2 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -403,59 +403,9 @@ object OldDeps {
403403

404404
object Catalyst {
405405
lazy val settings = antlr4Settings ++ Seq(
406-
antlr4PackageName in Antlr4 := Some("org.apache.spark.sql.catalyst.parser.ng"),
406+
antlr4PackageName in Antlr4 := Some("org.apache.spark.sql.catalyst.parser"),
407407
antlr4GenListener in Antlr4 := true,
408-
antlr4GenVisitor in Antlr4 := true,
409-
// ANTLR code-generation step.
410-
//
411-
// This has been heavily inspired by com.github.stefri.sbt-antlr (0.5.3). It fixes a number of
412-
// build errors in the current plugin.
413-
// Create Parser from ANTLR grammar files.
414-
sourceGenerators in Compile += Def.task {
415-
val log = streams.value.log
416-
417-
val grammarFileNames = Seq(
418-
"SparkSqlLexer.g",
419-
"SparkSqlParser.g")
420-
val sourceDir = (sourceDirectory in Compile).value / "antlr3"
421-
val targetDir = (sourceManaged in Compile).value / "antlr3"
422-
423-
// Create default ANTLR Tool.
424-
val antlr = new org.antlr.Tool
425-
426-
// Setup input and output directories.
427-
antlr.setInputDirectory(sourceDir.getPath)
428-
antlr.setOutputDirectory(targetDir.getPath)
429-
antlr.setForceRelativeOutput(true)
430-
antlr.setMake(true)
431-
432-
// Add grammar files.
433-
grammarFileNames.flatMap(gFileName => (sourceDir ** gFileName).get).foreach { gFilePath =>
434-
val relGFilePath = (gFilePath relativeTo sourceDir).get.getPath
435-
log.info("ANTLR: Grammar file '%s' detected.".format(relGFilePath))
436-
antlr.addGrammarFile(relGFilePath)
437-
// We will set library directory multiple times here. However, only the
438-
// last one has effect. Because the grammar files are located under the same directory,
439-
// We assume there is only one library directory.
440-
antlr.setLibDirectory(gFilePath.getParent)
441-
}
442-
443-
// Generate the parser.
444-
antlr.process()
445-
val errorState = org.antlr.tool.ErrorManager.getErrorState
446-
if (errorState.errors > 0) {
447-
sys.error("ANTLR: Caught %d build errors.".format(errorState.errors))
448-
} else if (errorState.warnings > 0) {
449-
sys.error("ANTLR: Caught %d build warnings.".format(errorState.warnings))
450-
}
451-
452-
// Return all generated java files.
453-
(targetDir ** "*.java").get.toSeq
454-
}.taskValue,
455-
// Include ANTLR tokens files.
456-
resourceGenerators in Compile += Def.task {
457-
((sourceManaged in Compile).value ** "*.tokens").get.toSeq
458-
}.taskValue
408+
antlr4GenVisitor in Antlr4 := true
459409
)
460410
}
461411

‎project/plugins.sbt‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,6 @@ libraryDependencies += "org.ow2.asm" % "asm" % "5.0.3"
2222

2323
libraryDependencies += "org.ow2.asm" % "asm-commons" % "5.0.3"
2424

25-
libraryDependencies += "org.antlr" % "antlr" % "3.5.2"
26-
27-
2825
// TODO I am not sure we want such a dep.
2926
resolvers += "simplytyped" at "http://simplytyped.github.io/repo/releases"
3027

‎python/pyspark/sql/utils.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,7 @@ def deco(*a, **kw):
5555
e.java_exception.getStackTrace()))
5656
if s.startswith('org.apache.spark.sql.AnalysisException: '):
5757
raise AnalysisException(s.split(': ', 1)[1], stackTrace)
58-
if s.startswith('org.apache.spark.sql.catalyst.parser.ng.ParseException: '):
58+
if s.startswith('org.apache.spark.sql.catalyst.parser.ParseException: '):
5959
raise ParseException(s.split(': ', 1)[1], stackTrace)
6060
if s.startswith('java.lang.IllegalArgumentException: '):
6161
raise IllegalArgumentException(s.split(': ', 1)[1], stackTrace)

‎sql/catalyst/pom.xml‎

Lines changed: 0 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -71,10 +71,6 @@
7171
<groupId>org.codehaus.janino</groupId>
7272
<artifactId>janino</artifactId>
7373
</dependency>
74-
<dependency>
75-
<groupId>org.antlr</groupId>
76-
<artifactId>antlr-runtime</artifactId>
77-
</dependency>
7874
<dependency>
7975
<groupId>org.antlr</groupId>
8076
<artifactId>antlr4-runtime</artifactId>
@@ -115,24 +111,6 @@
115111
</execution>
116112
</executions>
117113
</plugin>
118-
<plugin>
119-
<groupId>org.antlr</groupId>
120-
<artifactId>antlr3-maven-plugin</artifactId>
121-
<executions>
122-
<execution>
123-
<goals>
124-
<goal>antlr</goal>
125-
</goals>
126-
</execution>
127-
</executions>
128-
<configuration>
129-
<sourceDirectory>../catalyst/src/main/antlr3</sourceDirectory>
130-
<includes>
131-
<include>**/SparkSqlLexer.g</include>
132-
<include>**/SparkSqlParser.g</include>
133-
</includes>
134-
</configuration>
135-
</plugin>
136114
<plugin>
137115
<groupId>org.antlr</groupId>
138116
<artifactId>antlr4-maven-plugin</artifactId>

0 commit comments

Comments
 (0)