CDH

CDH Installation Notes

#!/bin/bash
#
# install cdh5 on centos7

# install some softwares
yum install -y zsh wget git mlocate iftop net-tools nfs-utils vim ntp epel-release

# zshrc
sh -c "$(curl -fsSL https://gitee.com/huaixwang/oh-my-zsh/raw/master/tools/install.sh)"

# add hosts resolving [all nodes]
echo '172.26.12.64 n12'  >> /etc/hosts
echo '172.26.11.189 n13' >> /etc/hosts
echo '172.26.11.187 n11' >> /etc/hosts
echo '172.26.11.191 n14' >> /etc/hosts

# ssh key configuration
ssh-keygen -t rsa -P "" -f /root/.ssh/id_rsa
cat /root/.ssh/id_rsa.pub  >> /root/.ssh/authorized_keys
# [master-node]
ssh-copy-id -i OTHER_NODES

# stop firewall (during testing) [all nodes]
systemctl stop firewalld && systemctl disable firewalld

# set selinux into disabled mode [all nodes]
sed -i "s#SELINUX=enforcing#SELINUX=disabled#" /etc/selinux/config

# install java sdk [all nodes]
wget https://mirrors.huaweicloud.com/java/jdk/8u191-b12/jdk-8u191-linux-x64.tar.gz -O /tmp/jdk-8u191.tar.gz && tar -xvf /tmp/jdk-8u191.tar.gz -C /usr/local/
mv /usr/local/jdk* /usr/local/jdk
rm -rf /tmp/jdk-8u191.tar.gz
echo 'export JAVA_HOME=/usr/local/jdk' >> /etc/profile
echo 'export PATH=$PATH:$JAVA_HOME/bin:$JAVA_HOME/sbin' >> /etc/profile
echo 'export CLASSPATH=.:$JAVA_HOME/lib/dt.jar:$JAVA_HOME/lib/tools.jar' >> /etc/profile
source /etc/profile

# ntp configuration   ps: make master-node as ntp server will be better [all nodes]
yum install -y ntp
sed -i "s/^server/#&/" /etc/ntp.conf
echo 'server ntp1.aliyun.com iburst' >> /etc/ntp.conf
systemctl start ntpd && systemctl enable ntpd
timedatectl set-timezone Asia/Shanghai
ntpdate ntp1.aliyun.com

# master-node installs MySql [master-node]
yum install -y https://dev.mysql.com/get/mysql57-community-release-el7-9.noarch.rpm
yum install -y mysql-community-server
systemctl start mysqld && systemctl enable mysqld
    # get MySql root's password from log
    grep password /var/log/mysqld.log |awk '{print$NF}'
    # initialize MySql
    mysql_secure_installation
    # highly recommended using mycli, install it with:
    yum install -y python3 &&  python3 -m pip install  mycli

mysql -uroot -p"YOUR_PASSWORD"
            # in case some errors about 'scm' user maybe happen latter
  mysql>    set global validate_password_policy=LOW
            set global validate_password_length=3
            set global validate_password_mixed_case_count=0
            set global validate_password_number_count=0
            set global validate_password_special_char_count=0

            CREATE DATABASE scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
            CREATE DATABASE amon DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
            CREATE DATABASE rman DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
            CREATE DATABASE hue DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
            CREATE DATABASE metastore DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
            CREATE DATABASE sentry DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
            CREATE DATABASE nav DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
            CREATE DATABASE navms DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
            CREATE DATABASE oozie DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
            CREATE DATABASE hive DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;

            grant all on *.* to 'root'@'%' identified by 'YOUR_PASSWORD' with grant option;



# install Cloudera Manager Server and Agent [all-nodes]
# CM url: https://archive.cloudera.com/cm5/cm/5/
# CDH url: http://archive.cloudera.com/cdh5/parcels/5.15.0/
# or: https://gd.ley.best/opstools/
wget https://gd.ley.best/opstools/cloudera-manager-centos7-cm5.15.0_x86_64.tar.gz
wget https://gd.ley.best/opstools/CDH-5.15.0-1.cdh5.15.0.p0.21-el7.parcel
tar -xvf cloudera-manager-* -C /opt
# create scm user
useradd --system --home=/opt/cm-5.15.0/run/cloudera-scm-server/ --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm
# modify server_host's value as master-node's hostname
vi /opt/cm-5.15.0/etc/cloudera-scm-agent/config.ini
server_host=YOUR_MASTER_NODE_HOSTNAME
# sed -i "s/localhost/YOUR_MASTER_NODE_HOSTNAME/" /opt/cm-5.15.0/etc/cloudera-scm-agent/config.ini

# jdbc
wget https://downloads.mysql.com/archives/get/p/3/file/mysql-connector-java-5.1.47.tar.gz
tar -xvf mysql-connector-java-*
cp  mysql-connector-java-5.1.47/mysql-connector-java-5.1.47-bin.jar  /opt/cm-5.15.0/share/cmf/lib/mysql-connector-java.jar

# initialize cm database [master-node]
/opt/cm-5.15.0/share/cmf/schema/scm_prepare_database.sh mysql cm -h MASTER_NODE_HOSTNAME -uroot -pYOUR_MYSQL_PASSWORD --scm-host MASTER_NODE_HOSTNAME scm scm scm
#                                                       db_type::db_name::db_server::db_user::db_user_passwd::cm_server_host::db_name::db_user::db_passwd [attention with mysql's passwd policy]

# kernel setting
echo never > /sys/kernel/mm/transparent_hugepage/defrag
echo never > /sys/kernel/mm/transparent_hugepage/enabled
echo 10 > /proc/sys/vm/swappiness
echo 'echo never > /sys/kernel/mm/transparent_hugepage/defrag' >> /etc/rc.local
echo 'echo never > /sys/kernel/mm/transparent_hugepage/enabled' >> /etc/rc.local
echo 'echo 10 > /proc/sys/vm/swappiness' >> /etc/rc.local

# start CM Manager and Agent
[master-node]:  /opt/cm-5.15.0/etc/init.d/cloudera-scm-server start
[agent-node]:   /opt/cm-5.15.0/etc/init.d/cloudera-scm-agent start

# self-starting

# [master-node]
cp /opt/cm-5.15.0/etc/init.d/cloudera-scm-server /etc/init.d/cloudera-scm-server
chkconfig --add cloudera-scm-server
chkconfig cloudera-scm-server on
vi /etc/init.d/cloudera-scm-serve
    CMF_DEFAULTS=${CMF_DEFAULTS:-/etc/default}  -> CMF_DEFAULTS=${CMF_DEFAULTS:-/opt/cm-5.15.0/etc/default}
    export JAVA_HOME=/usr/local/jdk
service cloudera-scm-server start & systemctl start cloudera-scm-server.service

# [agent-node]
cp /opt/cm-5.15.0/etc/init.d/cloudera-scm-agent /etc/init.d/cloudera-scm-agent
chkconfig --add cloudera-scm-agent
chkconfig cloudera-scm-agent on
vi /etc/init.d/cloudera-scm-agent
    CMF_DEFAULTS=${CMF_DEFAULTS:-/etc/default}   -> CMF_DEFAULTS=${CMF_DEFAULTS:-/opt/cm-5.15.0/etc/default}
    export JAVA_HOME=/usr/local/jdk
systemctl deamon-reload
service cloudera-scm-agent start & systemctl start cloudera-scm-agent.service


netstat -lntup|grep 7180
# if it works then we can browse http://MASTER_NODE_IP:7180 with user_name "admin" and password "admin"
# install the free version with setup guidelines and done