searchusermenu
  • 发布文章
  • 消息中心
点赞
收藏
评论
分享
原创

Atlas Hook部署之HiveMetastore服务

2024-05-28 09:00:13
11
0

1,查看已有的hive.metastore服务

查看部署目录

[root@cth04 ~]# which hive
/usr/local/hive/bin/hive
 

查看进程

[root@cth04 ~]# ps -ef | grep -i HiveMetaStore
hive 147304 1 0 6⽉26 ? 01:06:10 /usr/jdk64/current/bin/java xxx... org.apache.hadoop.util.RunJar /usr/local/hive/lib/hive-metastore-3.1.2.jar
org.apache.hadoop.hive.metastore.HiveMetaStore
 

HiveMetaStore服务是hive⽤户启动的,切换到hive⽤户操作

[root@cth04 ~]# sudo su -
[root@cth04 ~]# su - hive
[hive@cth04 ~]$
 

2,查看已有的keytabs配置

[hive@cth04 ~]$ cd /etc/security/keytabs/
[hive@cth04 keytabs]$ ll
total 28
-r-------- 1 hbase hbase 1298 Jun 26 21:15 hbase.keytab
-r-------- 1 hdfs hdfs 1922 Jun 26 20:57 hdfs.keytab
-r-------- 1 hive hive 1922 Jun 26 21:09 hive.keytab
2-r-------- 1 hdfs hdfs 1922 Jun 26 20:57 HTTP.keytab
-r-------- 1 mapred hadoop 1970 Jun 26 21:05 mapred.keytab
-r-------- 1 spark spark 650 Jun 26 21:12 spark.keytab
-r-------- 1 yarn hadoop 1922 Jun 26 21:05 yarn.keytab
 

klist查看凭据

[hive@cth04 keytabs]$ klist -kt /etc/security/keytabs/hive.keytab
 

kinit重新申请凭据

[hive@cth04 keytabs]$ kinit -kt /etc/security/keytabs/hive.keytab
 
[hive@cth04 keytabs]$ klist
 

3,atlas hook与hive集成的完整jar依赖

[hive@cth04 ~]$ which hive
/usr/local/hive/bin/hive
 
[hive@cth04 ~]$ cd /usr/local/hive
 
[hive@cth04 hive]$ ll
total 10308
-r-------- 1 root root 10471435 Jul 4 19:38 apache-atlas-2.2.0-hive-hook.tar.gz
drwxr-xr-x 3 root root 4096 Jul 5 17:10 apache-atlas-hive-hook-2.2.0
drwxr-xr-x 3 root root 4096 Jun 26 21:08 bin
drwxr-xr-x 2 root root 4096 Jun 27 16:27 conf
drwxr-xr-x 3 root root 4096 Jul 4 19:37 hook
......
-------Hive Hook JAR包存放位置
[hive@cth04 hive]$ ll hook/hive/
total 36
drwxr-xr-x 2 root root 4096 Jul 4 19:37 atlas-hive-plugin-impl
-rw-r--r-- 1 root root 17512 Jul 4 19:37 atlas-plugin-classloader-2.2.0.jar
-rw-r--r-- 1 root root 11562 Jul 4 19:37 hive-bridge-shim-2.2.0.jar
 

4,atlas hook与hive.client和hive.metastore服务集成

4.1 查看hive部署路径

[root@cth04 ~]# which hive
/usr/local/hive/bin/hive
 
[root@cth04 ~]# cd /usr/local/hive/conf/
 
[root@cth04 conf]# pwd
/usr/local/hive/conf
 
[root@cth04 conf]# ll conf/
......
-rw-r--r-- 1 root root 2913 Jun 26 21:08 hive-env.sh
-rw-r--r-- 1 root root 18160 Jun 26 21:08 hive-site.xml
......
 

4.2 增加jar包环境,修改hive-env.sh中HIVE_AUX_JARS_PATH

[root@cth04 conf]# cp hive-env.sh hive-env.sh_bak0705
[root@cth04 conf]# vim hive-env.sh
[root@cth04 conf]# cat /usr/local/hive/conf/hive-env.sh
......
export METASTORE_PORT=9083
#export
HADOOP_CLASSPATH=${TEZ_CONF_DIR}:${TEZ_JARS}/*:${TEZ_JARS}/lib/*:${HADOOP_CLASS
PATH}
export JAVA_HOME="/usr/jdk64/current"
#atlas hook
export HIVE_AUX_JARS_PATH=/usr/local/hive/hook/hive
 

4.3 增加启⽤hook配置

[root@cth04 conf]# cp hive-site.xml hive-site.xml_bak0705
[root@cth04 conf]# vim hive-site.xml
[root@cth04 conf]# cat hive-site.xml | grep -i -C1 atlas
<!--hive.client客户端&hive.hiveserver2服务端 启⽤atlas hook-->
<property>
<name>hive.exec.post.hooks</name>
<value>org.apache.atlas.hive.hook.HiveHook</value>
</property>
<!--hive.metastore服务端 启⽤atlas hook-->
<property>
<name>hive.metastore.event.listeners</name>
<value>org.apache.atlas.hive.hook.HiveMetastoreHook</value>
</property>
 

4.4 增加atlas hook相关配置,连接kafka集群等

特别注意atlas.jaas.KafkaClient.option.principal中keytab相关配置,要与实际服务器上配置⼀致
[root@cth04 conf]# ll | grep -i atlas
-r-------- 1 root root 2552 7⽉ 5 15:31 atlas-application.properties
[root@cth04 conf]# chmod +777 atlas-application.properties
[root@cth04 conf]# ll | grep -i atlas
-rwxrwxrwx 1 root root 2549 7⽉ 5 20:16 atlas-application.properties
 

4.5 重启hive.metastore服务端

对应的log⽂件为/var/log/hive/hivemetastore.log
[hive@cth04 ~]$ cd /var/log/hive
[hive@cth04 hive]$ pwd
/var/log/hive
[hive@cth04 hive]$ ll
-rw-r--r-- 1 hive hadoop 1532910 Jul 5 20:00 hivemetastore.log
-rw-r--r-- 1 hive hadoop 626 Jul 5 05:20 hiveserver2.log
[hive@cth04 hive]$ nohup hive --service metastore &
[hive@cth04 hive]$ ps -ef | grep -i metastore
[hive@cth04 hive]$ more hivemetastore.log | grep -i -C2 kafka
 

5,连接指定的hive.metastore来测试hive ddl

hive --hiveconf hive.metastore.uris=thrift://cth04:9083
hive --hiveconf hive.metastore.uris=thrift://cth05:9083
---------------------------
[hive@cth04 hive]$ hive --hiveconf hive.metastore.uris=thrift://cth04:9083
hive> show databases;
OK
default
hadoop_default
test_0315
test_0321
Time taken: 0.93 seconds, Fetched: 4 row(s)
hive>
create table default.student(sno int,name varchar(50)) comment '学⽣信息表1231233123';
OK
Time taken: 4.322 seconds
hive> create table default.student2 as select * from default.student;
hive> show databases;
OK
default
hadoop_default
luban_test
test_0315
test_0321
8hive> drop database luban_test;
OK
Time taken: 1.961 seconds
hive> create database luban_test;
OK
Time taken: 0.69 seconds

6,查看指定的hive.metastore的log

# tail -f /var/log/hive/hivemetastore.log
0条评论
作者已关闭评论
s****m
3文章数
0粉丝数
s****m
3 文章 | 0 粉丝
s****m
3文章数
0粉丝数
s****m
3 文章 | 0 粉丝
原创

Atlas Hook部署之HiveMetastore服务

2024-05-28 09:00:13
11
0

1,查看已有的hive.metastore服务

查看部署目录

[root@cth04 ~]# which hive
/usr/local/hive/bin/hive
 

查看进程

[root@cth04 ~]# ps -ef | grep -i HiveMetaStore
hive 147304 1 0 6⽉26 ? 01:06:10 /usr/jdk64/current/bin/java xxx... org.apache.hadoop.util.RunJar /usr/local/hive/lib/hive-metastore-3.1.2.jar
org.apache.hadoop.hive.metastore.HiveMetaStore
 

HiveMetaStore服务是hive⽤户启动的,切换到hive⽤户操作

[root@cth04 ~]# sudo su -
[root@cth04 ~]# su - hive
[hive@cth04 ~]$
 

2,查看已有的keytabs配置

[hive@cth04 ~]$ cd /etc/security/keytabs/
[hive@cth04 keytabs]$ ll
total 28
-r-------- 1 hbase hbase 1298 Jun 26 21:15 hbase.keytab
-r-------- 1 hdfs hdfs 1922 Jun 26 20:57 hdfs.keytab
-r-------- 1 hive hive 1922 Jun 26 21:09 hive.keytab
2-r-------- 1 hdfs hdfs 1922 Jun 26 20:57 HTTP.keytab
-r-------- 1 mapred hadoop 1970 Jun 26 21:05 mapred.keytab
-r-------- 1 spark spark 650 Jun 26 21:12 spark.keytab
-r-------- 1 yarn hadoop 1922 Jun 26 21:05 yarn.keytab
 

klist查看凭据

[hive@cth04 keytabs]$ klist -kt /etc/security/keytabs/hive.keytab
 

kinit重新申请凭据

[hive@cth04 keytabs]$ kinit -kt /etc/security/keytabs/hive.keytab
 
[hive@cth04 keytabs]$ klist
 

3,atlas hook与hive集成的完整jar依赖

[hive@cth04 ~]$ which hive
/usr/local/hive/bin/hive
 
[hive@cth04 ~]$ cd /usr/local/hive
 
[hive@cth04 hive]$ ll
total 10308
-r-------- 1 root root 10471435 Jul 4 19:38 apache-atlas-2.2.0-hive-hook.tar.gz
drwxr-xr-x 3 root root 4096 Jul 5 17:10 apache-atlas-hive-hook-2.2.0
drwxr-xr-x 3 root root 4096 Jun 26 21:08 bin
drwxr-xr-x 2 root root 4096 Jun 27 16:27 conf
drwxr-xr-x 3 root root 4096 Jul 4 19:37 hook
......
-------Hive Hook JAR包存放位置
[hive@cth04 hive]$ ll hook/hive/
total 36
drwxr-xr-x 2 root root 4096 Jul 4 19:37 atlas-hive-plugin-impl
-rw-r--r-- 1 root root 17512 Jul 4 19:37 atlas-plugin-classloader-2.2.0.jar
-rw-r--r-- 1 root root 11562 Jul 4 19:37 hive-bridge-shim-2.2.0.jar
 

4,atlas hook与hive.client和hive.metastore服务集成

4.1 查看hive部署路径

[root@cth04 ~]# which hive
/usr/local/hive/bin/hive
 
[root@cth04 ~]# cd /usr/local/hive/conf/
 
[root@cth04 conf]# pwd
/usr/local/hive/conf
 
[root@cth04 conf]# ll conf/
......
-rw-r--r-- 1 root root 2913 Jun 26 21:08 hive-env.sh
-rw-r--r-- 1 root root 18160 Jun 26 21:08 hive-site.xml
......
 

4.2 增加jar包环境,修改hive-env.sh中HIVE_AUX_JARS_PATH

[root@cth04 conf]# cp hive-env.sh hive-env.sh_bak0705
[root@cth04 conf]# vim hive-env.sh
[root@cth04 conf]# cat /usr/local/hive/conf/hive-env.sh
......
export METASTORE_PORT=9083
#export
HADOOP_CLASSPATH=${TEZ_CONF_DIR}:${TEZ_JARS}/*:${TEZ_JARS}/lib/*:${HADOOP_CLASS
PATH}
export JAVA_HOME="/usr/jdk64/current"
#atlas hook
export HIVE_AUX_JARS_PATH=/usr/local/hive/hook/hive
 

4.3 增加启⽤hook配置

[root@cth04 conf]# cp hive-site.xml hive-site.xml_bak0705
[root@cth04 conf]# vim hive-site.xml
[root@cth04 conf]# cat hive-site.xml | grep -i -C1 atlas
<!--hive.client客户端&hive.hiveserver2服务端 启⽤atlas hook-->
<property>
<name>hive.exec.post.hooks</name>
<value>org.apache.atlas.hive.hook.HiveHook</value>
</property>
<!--hive.metastore服务端 启⽤atlas hook-->
<property>
<name>hive.metastore.event.listeners</name>
<value>org.apache.atlas.hive.hook.HiveMetastoreHook</value>
</property>
 

4.4 增加atlas hook相关配置,连接kafka集群等

特别注意atlas.jaas.KafkaClient.option.principal中keytab相关配置,要与实际服务器上配置⼀致
[root@cth04 conf]# ll | grep -i atlas
-r-------- 1 root root 2552 7⽉ 5 15:31 atlas-application.properties
[root@cth04 conf]# chmod +777 atlas-application.properties
[root@cth04 conf]# ll | grep -i atlas
-rwxrwxrwx 1 root root 2549 7⽉ 5 20:16 atlas-application.properties
 

4.5 重启hive.metastore服务端

对应的log⽂件为/var/log/hive/hivemetastore.log
[hive@cth04 ~]$ cd /var/log/hive
[hive@cth04 hive]$ pwd
/var/log/hive
[hive@cth04 hive]$ ll
-rw-r--r-- 1 hive hadoop 1532910 Jul 5 20:00 hivemetastore.log
-rw-r--r-- 1 hive hadoop 626 Jul 5 05:20 hiveserver2.log
[hive@cth04 hive]$ nohup hive --service metastore &
[hive@cth04 hive]$ ps -ef | grep -i metastore
[hive@cth04 hive]$ more hivemetastore.log | grep -i -C2 kafka
 

5,连接指定的hive.metastore来测试hive ddl

hive --hiveconf hive.metastore.uris=thrift://cth04:9083
hive --hiveconf hive.metastore.uris=thrift://cth05:9083
---------------------------
[hive@cth04 hive]$ hive --hiveconf hive.metastore.uris=thrift://cth04:9083
hive> show databases;
OK
default
hadoop_default
test_0315
test_0321
Time taken: 0.93 seconds, Fetched: 4 row(s)
hive>
create table default.student(sno int,name varchar(50)) comment '学⽣信息表1231233123';
OK
Time taken: 4.322 seconds
hive> create table default.student2 as select * from default.student;
hive> show databases;
OK
default
hadoop_default
luban_test
test_0315
test_0321
8hive> drop database luban_test;
OK
Time taken: 1.961 seconds
hive> create database luban_test;
OK
Time taken: 0.69 seconds

6,查看指定的hive.metastore的log

# tail -f /var/log/hive/hivemetastore.log
文章来自个人专栏
文章 | 订阅
0条评论
作者已关闭评论
作者已关闭评论
0
0