hadoop单机部署

记录如何单机部署hadoop，并启动HDFS文件系统

# 1.安装java环境

略过

[root@summer ~]# java -version
java version "1.8.0_171"
Java(TM) SE Runtime Environment (build 1.8.0_171-b11)
Java HotSpot(TM) 64-Bit Server VM (build 25.171-b11, mixed mode)
[root@summer ~]#

1
2
3
4
5

# 2.整体目录结构

1.上传hadoop-2.7.3.tar.gz至服务器/home/summer目录下并解压

[root@master summer]# tar -zxvf hadoop-2.7.3.tar.gz
[root@master summer]# mv hadoop-2.7.3 hadoop
[root@master summer]# chown -R summer:summer /home/summer/hadoop
[root@master summer]# ll
drwxr-xr-x  9 summer summer       149 8月  18 2016 hadoop

1
2
3
4
5

# 3.配置环境变量

[root@summer hadoop]# cat  ~/.bashrc
# .bashrc

# User specific aliases and functions

alias rm='rm -i'
alias cp='cp -i'
alias mv='mv -i'

# Source global definitions
if [ -f /etc/bashrc ]; then
        . /etc/bashrc
fi

export HADOOP_HOME=/home/summer/hadoop
export HADOOP_CONF_DIR=/home/summer/hadoop/etc/hadoop
export PATH=$PATH:$HADOOP_HOME/bin:$HADOOP_HOME/sbin 
[root@summer hadoop]#source  ~/.bashrc
[root@summer hadoop]# hadoop
Usage: hadoop [--config confdir] [COMMAND | CLASSNAME]
  CLASSNAME            run the class named CLASSNAME
 or
  where COMMAND is one of:
  fs                   run a generic filesystem user client
  version              print the version
  jar <jar>            run a jar file
                       note: please use "yarn jar" to launch
                             YARN applications, not this command.
  checknative [-a|-h]  check native hadoop and compression libraries availability
  distcp <srcurl> <desturl> copy file or directories recursively
  archive -archiveName NAME -p <parent path> <src>* <dest> create a hadoop archive
  classpath            prints the class path needed to get the
  credential           interact with credential providers
                       Hadoop jar and the required libraries
  daemonlog            get/set the log level for each daemon
  trace                view and modify Hadoop tracing settings

Most commands print help when invoked w/o parameters.
[root@summer hadoop]#

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39

# 4.修改配置文件

[root@summer hadoop]# ls
capacity-scheduler.xml  hadoop-env.sh               httpfs-env.sh            kms-env.sh            mapred-env.sh               ssl-server.xml.example
configuration.xsl       hadoop-metrics2.properties  httpfs-log4j.properties  kms-log4j.properties  mapred-queues.xml.template  yarn-env.cmd
container-executor.cfg  hadoop-metrics.properties   httpfs-signature.secret  kms-site.xml          mapred-site.xml.template    yarn-env.sh
core-site.xml           hadoop-policy.xml           httpfs-site.xml          log4j.properties      slaves                      yarn-site.xml
hadoop-env.cmd          hdfs-site.xml               kms-acls.xml             mapred-env.cmd        ssl-client.xml.example
[root@summer hadoop]# pwd
/home/summer/hadoop/etc/hadoop
[root@summer hadoop]#

1
2
3
4
5
6
7
8
9

core-site.xml

<configuration>
     <property>
         <name>fs.defaultFS</name>
         <value>hdfs://192.168.0.52:9000</value>                               
     </property>
     <!-- 指定hadoop临时目录 -->
     <property>
         <name>hadoop.tmp.dir</name>
         <value>/home/summer/hadoop/data/tmp</value>
     </property>
</configuration>

1
2
3
4
5
6
7
8
9
10
11

hdfs-site.xml

<!-- Put site-specific property overrides in this file. -->

<configuration>
      <property>
         <name>dfs.replication</name>
         <value>1</value>
      </property>
</configuration>

1
2
3
4
5
6
7
8

hadoop-env.sh

[root@master hadoop]# pwd
/home/summer/hadoop/etc/hadoop
[root@master hadoop]# vim hadoop-env.sh
# The java implementation to use.
export JAVA_HOME=/usr/java/latest/  //这里配置JAVA_HOME

1
2
3
4
5

# 5.格式化

[root@summer hadoop]# hadoop namenode -format
DEPRECATED: Use of this script to execute hdfs command is deprecated.
Instead use the hdfs command for it.

21/03/04 11:18:08 INFO namenode.NameNode: STARTUP_MSG: 
/************************************************************
STARTUP_MSG: Starting NameNode
STARTUP_MSG:   host = localhost/127.0.0.1
STARTUP_MSG:   args = [-format]
STARTUP_MSG:   version = 2.7.3
STARTUP_MSG:   classpath = /home/summer/hadoop/etc/hadoop:/home/summer/hadoop/share/hadoop/common/lib/jaxb-impl-2.2.3-1.jar:/home/summer/hadoop/sh...
...
...
21/03/04 11:18:09 INFO common.Storage: Storage directory /home/summer/hadoop/data/tmp/dfs/name has been successfully formatted.
21/03/04 11:18:09 INFO namenode.FSImageFormatProtobuf: Saving image file /home/summer/hadoop/data/tmp/dfs/name/current/fsimage.ckpt_0000000000000000000 using no compression
21/03/04 11:18:09 INFO namenode.FSImageFormatProtobuf: Image file /home/summer/hadoop/data/tmp/dfs/name/current/fsimage.ckpt_0000000000000000000 of size 351 bytes saved in 0 seconds.
21/03/04 11:18:09 INFO namenode.NNStorageRetentionManager: Going to retain 1 images with txid >= 0
21/03/04 11:18:09 INFO util.ExitUtil: Exiting with status 0
21/03/04 11:18:09 INFO namenode.NameNode: SHUTDOWN_MSG: 
/************************************************************
SHUTDOWN_MSG: Shutting down NameNode at localhost/127.0.0.1
************************************************************/

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23

# 6.通过脚本启动hdfs

[root@summer sbin]# ./start-dfs.sh 
Starting namenodes on [192.168.0.52]
192.168.0.52: starting namenode, logging to /home/summer/hadoop/logs/hadoop-root-namenode-summer.out
localhost: starting datanode, logging to /home/summer/hadoop/logs/hadoop-root-datanode-summer.out
Starting secondary namenodes [0.0.0.0]
0.0.0.0: starting secondarynamenode, logging to /home/summer/hadoop/logs/hadoop-root-secondarynamenode-summer.out
[root@summer sbin]# jps
3409 NameNode
3973 DataNode
4725 SecondaryNameNode
9150 Jps
[root@summer hadoop]# netstat -antlp | grep 50070
tcp        0      0 0.0.0.0:50070           0.0.0.0:*               LISTEN      3409/java           
[root@summer bin]# ./hdfs dfs -mkdir /user
[root@summer bin]# ./hdfs dfs -mkdir /user/dream
[root@summer bin]# hadoop dfs -ls /
DEPRECATED: Use of this script to execute hdfs command is deprecated.
Instead use the hdfs command for it.

Found 1 items
drwxr-xr-x   - root supergroup          0 2021-03-04 11:22 /user
[root@summer bin]# hdfs dfs -ls /
Found 1 items
drwxr-xr-x   - root supergroup          0 2021-03-04 11:22 /user

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

# 7.浏览器验证即可

编辑

#hadoop #HDFS

上次更新: 3/5/2021, 6:35:04 PM

← ansible自动搭建elasticsearch集群挂载HDFS到本地目录→