forked from saikocat/spark-sql-kafka-avroflumeevent
-
Notifications
You must be signed in to change notification settings - Fork 0
/
build.sbt
67 lines (53 loc) · 2.73 KB
/
build.sbt
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
organization := "com.cloudera"
version := "0.1.0"
scalaVersion := "2.10.4"
scalacOptions ++= Seq("-unchecked", "-deprecation", "-feature")
javacOptions ++= Seq("-source", "1.7", "-target", "1.7")
resolvers += "Cloudera Hadoop Repo" at "https://repository.cloudera.com/content/repositories/releases"
resolvers += "Cloudera Repos" at "http://repository.cloudera.com/cloudera/cloudera-repos/"
resolvers += "Concurrent Jars" at "http://conjars.org/repo"
libraryDependencies ++= Seq(
"org.apache.flume" % "flume-ng-core" % "1.5.0" % "provided",
"org.apache.flume" % "flume-ng-sdk" % "1.5.0" % "provided",
"org.apache.flume" % "flume-ng-tests" % "1.5.0" % "test",
"org.apache.avro" % "avro" % "1.7.6-cdh5.4.7" % "provided",
"org.apache.kafka" %% "kafka" % "0.8.2.2" % "provided"
exclude("javax.jms", "jms")
exclude("com.sun.jdmk", "jmxtools")
exclude("com.sun.jmx", "jmxri")
exclude("org.slf4j", "slf4j-simple"),
"org.apache.spark" %% "spark-streaming" % "1.3.0-cdh5.4.7" % "provided",
"org.apache.spark" %% "spark-streaming-kafka" % "1.3.0-cdh5.4.7" % "provided" ,
"org.apache.spark" %% "spark-sql" % "1.3.0-cdh5.4.7" % "provided",
"org.apache.spark" %% "spark-hive" % "1.3.0-cdh5.4.7" % "provided",
"org.eclipse.jetty.orbit" % "javax.servlet" % "3.0.0.v201112011016" % "provided,test,compile" artifacts Artifact("javax.servlet", "jar", "jar"),
"com.beust" % "jcommander" % "1.48",
"junit" % "junit" % "4.12" % "test",
"org.scalatest" %% "scalatest" % "2.2.6" % "test",
"org.apache.curator" % "curator-test" % "3.1.0" % "test",
"org.apache.kafka" %% "kafka" % "0.8.2.2" % "provided" classifier "test"
exclude("javax.jms", "jms")
exclude("com.sun.jdmk", "jmxtools")
exclude("com.sun.jmx", "jmxri")
exclude("org.slf4j", "slf4j-simple")
)
assemblyMergeStrategy in assembly := {
case PathList("org", "apache", "spark", "unused", "UnusedStubClass.class") => MergeStrategy.discard
case x =>
val oldStrategy = (assemblyMergeStrategy in assembly).value
oldStrategy(x)
}
mergeStrategy in assembly := {
case m if m.toLowerCase.endsWith("manifest.mf") => MergeStrategy.discard
case m if m.toLowerCase.matches("meta-inf.*\\.sf$") => MergeStrategy.discard
case "log4j.properties" => MergeStrategy.discard
case m if m.toLowerCase.startsWith("meta-inf/services/") => MergeStrategy.filterDistinctLines
case "reference.conf" => MergeStrategy.concat
case _ => MergeStrategy.first
}
assemblyExcludedJars in assembly <<= (fullClasspath in assembly) map { cp =>
val excludes = Set(
"javax.servlet-3.0.0.v201112011016.jar"
)
cp filter { jar => excludes(jar.data.getName) }
}