spark-shell使用mysql存储元数据

1.hive安装(依赖于hadoop) - 下载hive - 解压到/opt目录下 - 添加环境变量到~/.bashrc export HIVE_HOME=/opt/apache-hive-2.2.0-bin export PATH=$HIVE_HOME/bin:$PATH 2.hive-env.sh添加

HADOOP_HOME=/opt/hadoop-2.7.4

3.hive-site.xml添加[server端，hive目录下]

<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
    <name>javax.jdo.option.ConnectionURL</name>
    <value>jdbc:mysql://10.68.2.55:3307/hivedb?createDatabaseIfNotExist=true&characterEncoding=utf8&useSSL=false</value> #mysql连接信息
    <description>
      JDBC connect string for a JDBC metastore.
      To use SSL to encrypt/authenticate the connection, provide database-specific SSL flag in the connection URL.
      For example, jdbc:postgresql://myhost/db?ssl=true for postgres database.
    </description>
  </property>

<property>
    <name>javax.jdo.option.ConnectionDriverName</name>
    <value>com.mysql.jdbc.Driver</value>
    <description>Driver class name for a JDBC metastore</description>
</property>

<property>
    <name>javax.jdo.option.ConnectionUserName</name>
    <value>root</value>
    <description>Username to use against metastore database</description>
</property>

<property>
    <name>javax.jdo.option.ConnectionPassword</name>
    <value>hello</value>
    <description>password to use against metastore database</description>
</property>

<property>
    <name>hive.metastore.warehouse.dir</name>
    <value>hdfs://10.68.2.55:9000/user/hive/warehouse</value>
    <description>location of default database for the warehouse</description>
  </property>
</configuration>

4.hive-site.xml[client端，spark目录下]

<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
<configuration>
<property>
    <name>hive.metastore.uris</name>
    <value>thrift://10.68.2.55:8000</value>
    <description>Thrift URI for the remote metastore. Used by metastore client to connect to remote metastore.</description>
  </property>
  <property>
  <name>spark.sql.warehouse.dir</name>
  <value>/user/hive/warehouse</value> #hdfs dfs -mkdir -p /usr/hive/warehouse
</property>
</configuration>

5.下载mysql-connector-java-5.1.44-bin.jar，放到/opt/apache-hive-2.2.0-bin/lib目录下 6.

schematool -dbType mysql -initSchema #初始化
hive --service metastore -p <port>  #启动服务
启动spark-shell将会将元数据存储到mysql