-
Notifications
You must be signed in to change notification settings - Fork 1
/
setup_hadoop.sh
executable file
·67 lines (55 loc) · 2.07 KB
/
setup_hadoop.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
#!/bin/bash
## Author: Shaikh Aquib
##
## Purpose: Used to download and setup Hadoop 3.3.4 and OpenJDK8 on Debian GNU/Linux
## and Debian/Ubuntu based systems.
##
## Usage: Execute this script with a non-root user having sudo priviledges
## For setting up basic environment and adding user to sudo execute
## setup_debian.sh provided along with this.
##
TASK_COMPLETION_MSG="+++++++++++++ DONE +++++++++++++\n"
HADOOP_HOME=$PWD/hadoop-3.3.4
BASH_PROFILE=/home/$USER/.bashrc
HADOOP_TAR_FILE=$HADOOP_HOME.tar.gz
JAVA_PKG_FILE=$PWD/openlogic-openjdk-8u352-b08-linux-x64-deb.deb
HADOOP_CONF_DIR=$HADOOP_HOME/etc/hadoop
echo "[+] Downloading Hadoop"
if ! test -f $HADOOP_TAR_FILE; then
wget https://dlcdn.apache.org/hadoop/common/hadoop-3.3.4/hadoop-3.3.4.tar.gz
fi
printf $TASK_COMPLETION_MSG
echo "[+] Downloading and Install JDK8"
if ! test -f $JAVA_PKG_FILE; then
wget https://builds.openlogic.com/downloadJDK/openlogic-openjdk/8u352-b08/openlogic-openjdk-8u352-b08-linux-x64-deb.deb
fi
sudo dpkg -i $JAVA_PKG_FILE
sudo apt -f install
printf $TASK_COMPLETION_MSG
echo "[+] Extracting files"
if ! test -d $HADOOP_HOME; then
tar -xvzf $HADOOP_TAR_FILE
fi
printf $TASK_COMPLETION_MSG
echo "[+] Setting up environment variables"
JAVA_HOME="$(dirname $(dirname $(readlink -f $(which java))))"
echo "export JAVA_HOME=$JAVA_HOME" >> $HADOOP_CONF_DIR/hadoop-env.sh
printf $TASK_COMPLETION_MSG
echo "[+] Setting SSH to run without passphrase"
ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
chmod 0600 ~/.ssh/authorized_keys
echo "[+] Configuring Core Site"
cat core-site.xml > $HADOOP_CONF_DIR/core-site.xml
printf $TASK_COMPLETION_MSG
echo "[+] Configuring HDFS"
cat hdfs-site.xml > $HADOOP_CONF_DIR/hdfs-site.xml
printf $TASK_COMPLETION_MSG
echo "[+] Configuring Map Reduce"
cat mapred-site.xml > $HADOOP_CONF_DIR/mapred-site.xml
printf $TASK_COMPLETION_MSG
echo "[+] Configuring YARN"
cat yarn-site.xml > $HADOOP_CONF_DIR/yarn-site.xml
printf $TASK_COMPLETION_MSG
echo "[+] Format the namenode"
$HADOOP_HOME/bin/hdfs namenode -format