Cloudera Manager 6構築メモ

事前準備

echo never > /sys/kernel/mm/transparent_hugepage/defrag;
echo never > /sys/kernel/mm/transparent_hugepage/enabled;

echo "echo never > /sys/kernel/mm/transparent_hugepage/defrag" >> /etc/rc.local;
echo "echo never > /sys/kernel/mm/transparent_hugepage/enabled" >> /etc/rc.local;

systemctl disable firewalld;
systemctl stop firewalld;

echo 'XXX.XXX.XXX.XXX hosta' >> /etc/hosts;
echo 'XXX.XXX.XXX.XXX hostb' >> /etc/hosts;
echo 'XXX.XXX.XXX.XXX hostc' >> /etc/hosts;

yum install java-11-openjdk-devel -y;

cat > /etc/profile.d/java11.sh <<EOF
export JAVA_HOME=$(dirname $(dirname $(readlink $(readlink $(which javac)))))
export PATH=\$PATH:\$JAVA_HOME/bin
export CLASSPATH=.:\$JAVA_HOME/jre/lib:\$JAVA_HOME/lib:\$JAVA_HOME/lib/tools.jar
EOF

source /etc/profile.d/java11.sh;

echo 'vm.swappiness = 10' > /etc/sysctl.d/swappiness.conf

sysctl -p;

yum install epel-release -y;

yum install wget -y;

timedatectl set-timezone UTC;

vi /etc/chrony.conf

/etc/chrony.conf

server ntp.nict.jp iburst

cloudera-manager 手動インストール

wget https://archive.cloudera.com/cm6/6.2.0/redhat7/yum/cloudera-manager.repo -P /etc/yum.repos.d/;

rpm --import https://archive.cloudera.com/cm6/6.2.0/redhat7/yum/RPM-GPG-KEY-cloudera;

yum install cloudera-manager-daemons cloudera-manager-agent cloudera-manager-server -y;

yum install mariadb-server -y;

systemctl stop mariadb;

vi /etc/my.cnf;

/etc/my.cnf

[mysqld]
datadir=/var/lib/mysql
socket=/var/lib/mysql/mysql.sock
transaction-isolation = READ-COMMITTED
# Disabling symbolic-links is recommended to prevent assorted security risks;
# to do so, uncomment this line:
symbolic-links = 0
# Settings user and group are ignored when systemd is used.
# If you need to run mysqld under a different user or group,
# customize your systemd unit file for mariadb according to the
# instructions in http://fedoraproject.org/wiki/Systemd

key_buffer = 16M
key_buffer_size = 32M
max_allowed_packet = 32M
thread_stack = 256K
thread_cache_size = 64
query_cache_limit = 8M
query_cache_size = 64M
query_cache_type = 1

max_connections = 550
#expire_logs_days = 10
#max_binlog_size = 100M

#log_bin should be on a disk with enough free space.
#Replace '/var/lib/mysql/mysql_binary_log' with an appropriate path for your
#system and chown the specified folder to the mysql user.
log_bin=/var/lib/mysql/mysql_binary_log

#In later versions of MariaDB, if you enable the binary log and do not set
#a server_id, MariaDB will not start. The server_id must be unique within
#the replicating group.
server_id=1

binlog_format = mixed

read_buffer_size = 2M
read_rnd_buffer_size = 16M
sort_buffer_size = 8M
join_buffer_size = 8M

# InnoDB settings
innodb_file_per_table = 1
innodb_flush_log_at_trx_commit  = 2
innodb_log_buffer_size = 64M
innodb_buffer_pool_size = 4G
innodb_thread_concurrency = 8
innodb_flush_method = O_DIRECT
innodb_log_file_size = 512M

[mysqld_safe]
log-error=/var/log/mariadb/mariadb.log
pid-file=/var/run/mariadb/mariadb.pid

#
# include all files from the config directory
#
!includedir /etc/my.cnf.d

systemctl enable mariadb;

systemctl start mariadb;

/usr/bin/mysql_secure_installation;

[...]
Enter current password for root (enter for none):
OK, successfully used password, moving on...
[...]
Set root password? [Y/n] Y
New password:
Re-enter new password:
[...]
Remove anonymous users? [Y/n] Y
[...]
Disallow root login remotely? [Y/n] N
[...]
Remove test database and access to it [Y/n] Y
[...]
Reload privilege tables now? [Y/n] Y
[...]
All done!  If you've completed all of the above steps, your MariaDB
installation should now be secure.

Thanks for using MariaDB!'

wget https://dev.mysql.com/get/Downloads/Connector-J/mysql-connector-java-5.1.46.tar.gz;

tar zxvf mysql-connector-java-5.1.46.tar.gz;

sudo mkdir -p /usr/share/java/;

cd mysql-connector-java-5.1.46;

cp mysql-connector-java-5.1.46-bin.jar /usr/share/java/mysql-connector-java.jar;

mysql -u root -p;

CREATE DATABASE scm DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON scm.\* TO 'scm'@'%' IDENTIFIED BY '[@@PASSWORD@@]';

CREATE DATABASE amon DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON amon.\* TO 'amon'@'%' IDENTIFIED BY '[@@PASSWORD@@]';

CREATE DATABASE rman DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON rman.\* TO 'rman'@'%' IDENTIFIED BY '[@@PASSWORD@@]';

CREATE DATABASE hue DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON hue.\* TO 'hue'@'%' IDENTIFIED BY '[@@PASSWORD@@]';

CREATE DATABASE metastore DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON metastore.\* TO 'hive'@'%' IDENTIFIED BY '[@@PASSWORD@@]';

CREATE DATABASE sentry DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON sentry.\* TO 'sentry'@'%' IDENTIFIED BY '[@@PASSWORD@@]';

CREATE DATABASE nav DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON nav.\* TO 'nav'@'%' IDENTIFIED BY '[@@PASSWORD@@]';

CREATE DATABASE navms DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON navms.\* TO 'navms'@'%' IDENTIFIED BY '[@@PASSWORD@@]';

CREATE DATABASE oozie DEFAULT CHARACTER SET utf8 DEFAULT COLLATE utf8_general_ci;
GRANT ALL ON oozie.\* TO 'oozie'@'%' IDENTIFIED BY '[@@PASSWORD@@]';

SHOW DATABASES;

exit;

/opt/cloudera/cm/schema/scm_prepare_database.sh mysql scm scm;

systemctl start cloudera-scm-server;

tail -f /var/log/cloudera-scm-server/cloudera-scm-server.log