Hadoop (1) Ali Cloud Hadoop Cluster Configuration

2.$HADOOP_HOME /sbin files and their functions

< /p>

3. Files in the $HADOOP_HOME/etc/hadoop directory and their functions

4.$ HADOOP_HOME/lib directory

This directory stores the jar packages that Hadoop depends on when it runs. Hadoop will add all the jars under the lib directory to the classpath during execution.

5.$HADOOP_HOME/logs directory

This directory stores Hadoop running logs. Viewing the logs is very helpful for finding Hadoop running errors.

6.$HADOOP_HOME/include directory
The programming library header files provided to the outside (the specific dynamic library and static library are in the lib directory), these header files are used Defined in C++, usually used for C++ programs to access HDFS or to write MapReduce programs.
7.$HADOOP_HOME/libexec directory
The directory where the shell configuration file used by each service pair is located, which can be used to configure basic information such as log output and startup parameters (such as JVM parameters).
8.$HADOOP_HOME/share directory The directory where the jar package compiled by each Hadoop module is located.

mkdir /bigdata
cd /bigdata
mkdir /app

vim /etc/hosts

127.0.0.1 localhost localhost.localdomain localhost4 localhost4.localdomain4

::1 localhost localhost.localdomain localhost6 localhost6.localdomain6 172.16.237.91 node01 172.16.237.90 node02 172.16.221.55 node03 span>

tar -zxvf hadoop-2.8.4.tar.gz- C /bigdata/app

ln -s /bigdata/app/hadoop-2.8.4 /usr/local/hadoop

 vim /etc/profile

export HADOOP_HOME=/usr/local/hadoop

export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop

export YARN_HOME=$HADOOP_HOME

export YARN_CONF_DIR=$HADOOP_HOME/etc/hadoop

export PATH=$PATH:${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin

source /etc/profile

export JAVA_HOME=/usr/local/jdk

cd /usr/local/hadoop

mkdir hdpdata

node02

node03

vim /etc/profile

export HADOOP_HOME=/usr/local/ hadoop

export HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop

export YARN_HOME=$HADOOP_HOME

export YARN_CONF_DIR=$HADOOP_HOME/etc/hadoop
export PATH=$PATH:${HADOOP_HOME}/bin:${HADOOP_HOME}/sbin

source /etc/profile

/usr/local/hadoop/sbin/hadoop-daemon.sh start journalnode

hdfs namenode -format

scp -r hdpdata [email protected]:/usr/local/hadoop

hdfs zkfc -formatZK

sbin/start-dfs.sh

sbin/start-yarn.sh

sbin/yarn-daemon.sh start resourcemanager

sbin/mr-jobhistory-daemon.sh start historyserver

hadoop fs -put /usr/local/hadoop/README.txt /

sbin/hadoop-daemon.sh stop namenode

sbin/hadoop-daemon.sh start namenode

sbin/yarn-daemon.sh stop resourcemanager

sbin/yarn-daemon.sh start resourcemanager

"1.0" encoding="UTF-8"?>

"text/xsl" href="configuration.xsl"?>











    

    

        fs.defaultFS

        hdfs://ns

    

    

    

        hadoop.tmp.dir

        /usr/local/hadoop/hdpdata/

        需要手动创建hdpdata目录

    

    

    

        ha.zookeeper.quorum

        node01:2181,node02:2181,node03:2181

        zookeeper地址，多个用逗号隔开

    

"1.0" encoding="UTF-8"?>

"text/xsl" href="configuration.xsl"?>











    

    

        dfs.nameservices

        ns

        指定hdfs的nameservice为ns，需要和core-site.xml中的保持一致

    

    

        dfs.ha.namenodes.ns

        nn1,nn2

        ns命名空间下有两个NameNode，逻辑代号，随便起名字，分别是nn1，nn2

    

    

        dfs.namenode.rpc-address.ns.nn1

        node01:9000

        nn1的RPC通信地址

    

    

        dfs.namenode.http-address.ns.nn1

        node01:50070

        nn1的http通信地址

    

    

        dfs.namenode.rpc-address.ns.nn2

        node02:9000

        nn2的RPC通信地址

    

    

        dfs.namenode.http-address.ns.nn2

        node02:50070

        nn2的http通信地址

    

    

    

        dfs.namenode.shared.edits.dir

        qjournal://node01:8485;node02:8485;node03:8485/ns

    

    

        dfs.journalnode.edits.dir

        /usr/local/hadoop/journaldata

        指定JournalNode在本地磁盘存放数据的位置

    

    

    

        dfs.ha.automatic-failover.enabled

        true

        开启NameNode失败自动切换

    

    

        dfs.client.failover.proxy.provider.ns

        org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider

        配置失败自动切换实现方式,使用内置的zkfc

    

    

        dfs.ha.fencing.methods

        

            sshfence

            shell(/bin/true)

        

        配置隔离机制，多个机制用换行分割，先执行sshfence，执行失败后执行shell(/bin/true)，/bin/true会直接返回0表示成功

    

    

        dfs.ha.fencing.ssh.private-key-files

        /root/.ssh/id_rsa

        使用sshfence隔离机制时需要ssh免登陆

    

    

        dfs.ha.fencing.ssh.connect-timeout

        30000

        配置sshfence隔离机制超时时间

    

    

    

        dfs.replication

        3

        默认block副本数为3，测试环境这里设置为1，注意生产环境一定要设置3个副本以上

    



    

        dfs.block.size

        134217728

        设置block大小是128M

    

   

    dfs.client.use.datanode.hostname
    true
    only cofig in clients

xml version="1.0"?>

xml-stylesheet type="text/xsl" href="configuration.xsl"?>









<configuration>

    <property>

        <name>mapreduce.framework.namename>

        <value>yarnvalue>

        <description>指定mr框架为yarn方式 description>

    property>

    

    <property>

        <name>mapreduce.jobhistory.addressname>

        <value>node02:10020value>

        <description>历史服务器端口号description>

    property>

    <property>

        <name>mapreduce.jobhistory.webapp.addressname>

        <value>node02:19888value>

        <description>历史服务器的WEB UI端口号description>

    property>

configuration>

java.io.IOException: There appears to be a gap in the edit log.  We expected txid 1, but got txid 2.

hadoop namenode -recover