hadoop安装 - 2877206/docs GitHub Wiki

单机

分布

rpm -e cloudera-manager-agent-5.8.1-1.cm581.p0.7.el7.x86_64 rpm -e cloudera-manager-daemons-5.8.1-1.cm581.p0.7.el7.x86_64 cloudera/parcels/CDH-5.8.0-1.cdh5.8.0.p0.42/lib/hive/lib sed -i "s/server_host=localhost/server_host=192.168.252.134/" /opt/cm-5.8.1/etc/cloudera-scm-agent/config.ini mkdir /opt/cm-5.8.1/run/cloudera-scm-agent /opt/cm-5.8.1/etc/init.d/cloudera-scm-server start scp -r 192.168.252.134:/opt/cm-5.8.1 /opt /opt/cm-5.8.1/etc/init.d/cloudera-scm-agent start useradd --system --home=/opt/cm-5.7.0/run/cloudera-scm-server --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm usermod -m -d /opt/cm-5.8.1/run/cloudera-scm-server cloudera-scm scm_prepare_database.sh database-type [options] database-name username password ./scm_prepare_database.sh mysql cm -hlocalhost -uroot -p20100328 --scm-host centos7-03 scm scm scm ./scm_prepare_database.sh mysql cm -hcentos7-03 -uroot -p20100328 --scm-host centos7-03 scm 20100328 yum remove cloudera-manager-agent-5.8.1-1.cm581.p0.7.el7.x86_64 yum --nogpgcheck localinstall cloudera-manager-daemons-.rpm && yum --nogpgcheck localinstall cloudera-manager-server-.rpm yum --nogpgcheck localinstall cloudera-manager-agent-package.*.x86_64.rpm yum --nogpgcheck localinstall cloudera-manager-daemons-5.8.1-1.cm581.p0.7.el7.x86_64.rpm && yum --nogpgcheck localinstall cloudera-manager-agent-5.8.1-1.cm581.p0.7.el7.x86_64.rpm yum --nogpgcheck localinstall cloudera-manager-server-5.8.1-1.cm581.p0.7.el7.x86_64.rpm vi /etc/cloudera-scm-agent/config.ini service cloudera-scm-server start service cloudera-scm-agent start tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log tail -f /var/log/cloudera-scm-agent/cloudera-scm-agent /etc/hosts reboot

yum remove cloudera-manager-agent-5.8.1-1.cm581.p0.7.el7.x86_64 && yum remove cloudera-manager-server-5.8.1-1.cm581.p0.7.el7.x86_64 && yum remove cloudera-manager-daemons-5.8.1-1.cm581.p0.7.el7.x86_64

vi /etc/security/limits.conf文件，增加如下内容：

  • hard nofile 65535
  • soft nofile 65535

useradd --system --home=/opt/cm-5.1.3/run/cloudera-scm-server/ --no-create-home --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm

cd /opt/cloudera/parcels/CDH-5.8.0-1.cdh5.8.0.p0.42/lib/hive/lib cp ~/software/mysql-connector-java-5.1.8.jar ./

ps -aef | grep java for x in cd /etc/init.d ; ls hadoop-* ; do sudo service $x stop ; done

http://192.168.252.134:7180/cmf/express-wizard/wizard ##################server################# vi /etc/hosts export JAVA_HOME=/usr/java/jdk1.7.0_67-cloudera export PATH=$JAVA_HOME/bin:$PATH systemctl stop firewalld systemctl disable firewalld vi /etc/ntp.conf

mysql -uroot -p drop database cm

##start service cloudera-scm-server stop yum remove cloudera-manager-agent-5.8.1-1.cm581.p0.7.el7.x86_64 && yum remove cloudera-manager-server-5.8.1-1.cm581.p0.7.el7.x86_64 && yum remove cloudera-manager-daemons-5.8.1-1.cm581.p0.7.el7.x86_64 cd ~/software yum --nogpgcheck localinstall cloudera-manager-daemons-.rpm && yum --nogpgcheck localinstall cloudera-manager-server-.rpm && yum --nogpgcheck localinstall cloudera-manager-agent-*.x86_64.rpm /usr/share/cmf/schema/scm_prepare_database.sh mysql cm -hlocalhost -uroot -p20100328 --scm-host centos7-03 scm 20100328 ##finish vi /etc/cloudera-scm-agent/config.ini service cloudera-scm-server start service cloudera-scm-agent start tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log

############Agent:##################### vi /etc/hosts export JAVA_HOME=/usr/java/jdk1.7.0_67-cloudera export PATH=$JAVA_HOME/bin:$PATH systemctl stop firewalld systemctl disable firewalld vi /etc/ntp.conf

##start service cloudera-scm-agent stop yum remove cloudera-manager-agent-5.8.1-1.cm581.p0.7.el7.x86_64 && yum remove cloudera-manager-daemons-5.8.1-1.cm581.p0.7.el7.x86_64 cd ~/software yum --nogpgcheck localinstall cloudera-manager-daemons-.rpm && yum --nogpgcheck localinstall cloudera-manager-agent-.x86_64.rpm ##finish vi /etc/cloudera-scm-agent/config.ini service cloudera-scm-agent start tail -f /var/log/cloudera-scm-agent/cloudera-scm-agent

./cloudera-agent.sh
ntp
tail -f /var/log/cloudera-scm-agent/cloudera-scm-agent.log


# getenforce

Disabled

sestatus -v

SELinux status: disabled

getenforce

Disabled

sestatus -v

SELinux status: disabled 在其余的节点上，把时间服务器的server指向master namenode节点

vi /etc/ntp.conf

systemctl start ntpd

systemctl enable ntpd

systemctl start mariadb

systemctl enable mariadb

#mysql_secure_installation

MariaDB [(none)]> create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci; Query OK, 1 row affected (0.00 sec)

MariaDB [(none)]> create database amon DEFAULT CHARSET utf8 COLLATE utf8_general_ci; Query OK, 1 row affected (0.00 sec)

MariaDB [(none)]> create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci; Query OK, 1 row affected (0.00 sec)

MariaDB [(none)]> create database monitor DEFAULT CHARSET utf8 COLLATE utf8_general_ci; Query OK, 1 row affected (0.00 sec)

MariaDB [(none)]> create database oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci; Query OK, 1 row affected (0.00 sec)

MariaDB [(none)]> grant all on . to root@"%" Identified by "20100328"; Query OK, 0 rows affected (0.00 sec)

create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci; create database monitor DEFAULT CHARSET utf8 COLLATE utf8_general_ci; grant all on . TO 'root'@'%' IDENTIFIED BY '20100328';

grant all privileges on hive.* to 'root'@'%' identified by '20100328' with grant option; grant all privileges on hive.* to 'root'@'%' identified by '20100328' with grant option;

grant all privileges on hive.* to 'hive'@'%' identified by '20100328' with grant option; grant all privileges on amon.* to 'scm'@'%' identified by '20100328' with grant option; grant all privileges on hue.* to 'scm'@'%' identified by '20100328' with grant option; grant all privileges on oozie.* to 'root'@'%' identified by '20100328' with grant option; flush privileges; grant all privileges on hive.* to 'scm'@'centos7-03' identified by '20100328' with grant option; grant all privileges on amon.* to 'scm'@'centos7-03' identified by '20100328' with grant option; grant all privileges on hue.* to 'scm'@'centos7-03' identified by '20100328' with grant option; grant all privileges on oozie.* to 'scm'@'centos7-03' identified by '20100328' with grant option; flush privileges;

grant all privileges on cm.* to 'scm'@'%' identified by '20100328' with grant option; flush privileges;

/usr/share/cmf/schema/scm_prepare_database.sh mysql -h centos7-02 -uroot -p --scm-host centos7-02 scm scm scm

OSError: [Errno 2] No such file or directory: '/run/cloudera-scm-agent/supervisor/include/55-MGMT.ACTIVITYMONITOR-test-db-connection.conf' [11/Sep/2016 04:23:10 +0000] 8663 MainThread agent INFO Retiring process 55-MGMT.ACTIVITYMONITOR-test-db-connection

[11/Sep/2016 06:33:16 +0000] 1832 Monitor-HostMonitor filesystem_map WARNING Failed to join worker process collecting filesystem usage. All nodev filesystems will have unknown usage until the worker process is no longer active. Current nodev filesystems: /dev/shm,/run,/sys/fs/cgroup,/run/user/0,/run/cloudera-scm-agent/process

echo 0 > /proc/sys/vm/swappiness echo never > /sys/kernel/mm/transparent_hugepage/defrag echo "echo 0 > /proc/sys/vm/swappiness" >>/etc/rc.d/rc.local echo "echo never > /sys/kernel/mm/transparent_hugepage/defrag" >>/etc/rc.d/rc.local

	Cloudera recommends setting /proc/sys/vm/swappiness to a maximum of 10. Current setting is 30. Use the sysctl command to change this setting at run time and edit /etc/sysctl.conf for this setting to be saved after a reboot. You can continue with installation, but Cloudera Manager might report that your hosts are unhealthy because they are swapping. The following hosts are affected:

Transparent Huge Page Compaction is enabled and can cause significant performance problems. Run "echo never > /sys/kernel/mm/transparent_hugepage/defrag" to disable this, then add the same command to an init script such as /etc/rc.local so it will be set upon system reboot. The following hosts are affected:

service hue stop service sqoop-metastore stop service sqoop2-server stop service flume-node stop service flume-master stop service flume-ng-agent stop service oozie stop service hiveserver2 stop service hive-metastore stop service hbase-thrift stop service hbase-master stop service hadoop-hbase-regionserver stop service hadoop-mapreduce-historyserver stop

sudo service hadoop-yarn-resourcemanager stop

sudo service hadoop-yarn-nodemanager stop

drop database hive; drop database hue; drop database oozie; create database hive DEFAULT CHARSET utf8 COLLATE utf8_general_ci;
create database hue DEFAULT CHARSET utf8 COLLATE utf8_general_ci; create database oozie DEFAULT CHARSET utf8 COLLATE utf8_general_ci;

http://hadoop.apache.org/docs/stable/hadoop-project-dist/hadoop-common/SingleCluster.html

http://www.dongcoder.com/detail-19216.html

1.Mysql --hive数据库,为什么使用latin1，后面安装异常会做说明 create database hive DEFAULT CHARSET latin1; --集群监控数据库 create database amon DEFAULT CHARSET utf8; --hue数据库 create database hue DEFAULT CHARSET utf8; --oozie数据库 create database oozie default charset utf8; --授权 grant all privileges on hive.* to 'scm'@'%' identified by '20100328' with grant option; grant all privileges on amon.* to 'scm'@'%' identified by '20100328' with grant option; grant all privileges on hue.* to 'scm'@'%' identified by '20100328' with grant option; grant all privileges on oozie.* to 'scm'@'%' identified by '20100328' with grant option; flush privileges; 2.wget http://central.maven.org/maven2/mysql/mysql-connector-java/5.1.8/mysql-connector-java-5.1.8.jar 3.wget http://archive.cloudera.com/cm5/redhat/7/x86_64/cm/5.8.1/RPMS/x86_64/cloudera-manager-daemons-5.8.1-1.cm581.p0.7.el7.x86_64.rpm

./scm_prepare_database.sh mysql -hlocalhost -uroot -p20100328YY scm scm cdh

5.yum -y install bind-utils psmisc libxslt cyrus-sasl-plain cyrus-sasl-gssapi fuse portmap fuse-libs httpd mod_ssl openssl-devel Python-psycopg2 MySQL-python yum -y install lsb

http://archive.cloudera.com/cdh5/parcels/5.8.0.42/

ls -l /var/lib/ |grep hadoop

chgrp -R httpfs /var/lib/hadoop-httpfs chown -R httpfs /var/lib/hadoop-httpfs chgrp -R hdfs /var/lib/hadoop-hdfs chown -R hdfs /var/lib/hadoop-hdfs chgrp -R kms /var/lib/hadoop-kms chown -R kms /var/lib/hadoop-kms chgrp -R mapred /var/lib/hadoop-mapreduce chown -R mapred /var/lib/hadoop-mapreduce chgrp -R yarn /var/lib/hadoop-yarn chgrp -R yarn /var/lib/hadoop-yarn

chmod 755 /var/lib/spark chown -R spark /var/lib/spark chgrp -R spark /var/lib/spark

chmod 755 /var/lib/hive chown -R hive /var/lib/hive chgrp -R hive /var/lib/hive

chmod 755 /var/lib/hue chown -R hue /var/lib/hue chgrp -R hue /var/lib/hue

chmod 755 /var/lib/oozie chown -R oozie /var/lib/oozie chgrp -R oozie /var/lib/oozie

chmod 755 /var/lib/zookeeper chown -R zookeeper /var/lib/zookeeper chgrp -R zookeeper /var/lib/zookeeper

useradd --system --home=/opt/cm-5.7.0/run/cloudera-scm-server --shell=/bin/false --comment "Cloudera SCM User" cloudera-scm