support subdirs

5ccb22d4 · Pengfei Xue · c10f9000 · 5ccb22d4
Commit 5ccb22d4 authored Jun 20, 2019 by Pengfei Xue
Hide whitespace changes
Inline Side-by-side

Showing with 2 additions and 0 deletions

Main.scala src/main/scala/com/gmei/data/dq/Main.scala +2 -0

No files found.
--- a/src/main/scala/com/gmei/data/dq/Main.scala
+++ b/src/main/scala/com/gmei/data/dq/Main.scala
@@ -11,6 +11,8 @@ object Main {
  def main(args: Array[String]) = {
    val spark = SparkSession.builder()
      .enableHiveSupport()
+      .config("spark.hive.mapred.supports.subdirectories","true")
+      .config("spark.hadoop.mapreduce.input.fileinputformat.input.dir.recursive","true")
      .appName("DataQualityMonitor")
      .getOrCreate()