close connection appropriatly, minior fix

datahub-project · Dec 21, 2015 · 0eddd8a · 0eddd8a
1 parent 05d54b3
commit 0eddd8a
Show file tree

Hide file tree

Showing 5 changed files with 15 additions and 13 deletions.
diff --git a/build.gradle b/build.gradle
@@ -70,6 +70,6 @@ subprojects {
                             "play_ebean"         : "com.typesafe.play:play-java-ebean_2.10:2.2.4",
                             "play_java_jdbc"     : "com.typesafe.play:play-java-jdbc_2.10:2.2.4",
                             "play_cache"         : "com.typesafe.play:play-cache_2.10:2.2.4",
-                            "hive_exec"          : "org.apache.hive:hive-exec:0.8.0"
+                            "hive_exec"          : "org.apache.hive:hive-exec:1.2.1"
   ]
 }
diff --git a/metadata-etl/src/main/java/metadata/etl/dataset/hive/HiveMetadataEtl.java b/metadata-etl/src/main/java/metadata/etl/dataset/hive/HiveMetadataEtl.java
@@ -26,7 +26,7 @@ public void extract()
     logger.info("In Hive metadata ETL, launch extract jython scripts");
     InputStream inputStream = classLoader.getResourceAsStream("jython/HiveExtract.py");
     //logger.info("before call scripts " + interpreter.getSystemState().argv);
-    super.interpreter.execfile(inputStream);
+    interpreter.execfile(inputStream);
     inputStream.close();
   }
 

diff --git a/metadata-etl/src/main/resources/jython/HdfsLoad.py b/metadata-etl/src/main/resources/jython/HdfsLoad.py
@@ -30,7 +30,7 @@ def load_metadata(self):
 
         LOAD DATA LOCAL INFILE '{source_file}'
         INTO TABLE stg_dict_dataset
-        FIELDS TERMINATED BY '\Z' ESCAPED BY '\0'
+        FIELDS TERMINATED BY '\Z' ESCAPED BY '\\'
         (`name`, `schema`, properties, fields, urn, source, sample_partition_full_path, source_created_time, source_modified_time)
         SET db_id = {db_id},
         -- TODO storage_type = 'Avro',

diff --git a/metadata-etl/src/main/resources/jython/HiveExtract.py b/metadata-etl/src/main/resources/jython/HiveExtract.py
@@ -301,10 +301,10 @@ def get_all_databases(self):
   e = HiveExtract()
   e.conn_hms = zxJDBC.connect(jdbc_url, username, password, jdbc_driver)
 
-  e.databases = e.get_all_databases()
-  print 'Process databases : '
-  print e.databases
-
-
-  e.run(args[Constant.HIVE_SCHEMA_JSON_FILE_KEY], None)
-  e.conn_hms.close()
+  try:
+    e.databases = e.get_all_databases()
+    print 'Process databases : '
+    print e.databases
+    e.run(args[Constant.HIVE_SCHEMA_JSON_FILE_KEY], None)
+  finally:
+    e.conn_hms.close()
diff --git a/metadata-etl/src/main/resources/jython/HiveLoad.py b/metadata-etl/src/main/resources/jython/HiveLoad.py
@@ -158,6 +158,8 @@ def load_field(self):
   l.db_id = args[Constant.DB_ID_KEY]
   l.wh_etl_exec_id = args[Constant.WH_EXEC_ID_KEY]
   l.conn_mysql = zxJDBC.connect(JDBC_URL, username, password, JDBC_DRIVER)
-  l.load_metadata()
-  # l.load_field()
-  l.conn_mysql.close()
+  try:
+    l.load_metadata()
+    # l.load_field()
+  finally:
+    l.conn_mysql.close()