複数サーバでのオペレーションスクリプト - foolmacky/DevelopDocs GitHub Wiki

■ 基本的なtmuxのコマンド

  • デタッチ ctrl+b + d
  • アタッチ tmux a
  • プロセスが複数ある場合 tmux ls でプロセスがリストアップされる
  • プロセス0を指定してアタッチ tmux a -t 0

■ 設定ファイル

cat .tmux.conf
setw -g mode-mouse on
set -g mouse-select-pane on
set-option -g default-terminal screen-256color
set -g terminal-overrides 'xterm:colors=256'

bind S set-window-option synchronize-panes
  • ctrl+b S で入力モード(paneすべて同じコマンドか、独立か)切り替えが可能

■ 起動スクリプト

  • 複数のサーバに接続する場合は、予め接続対象のサーバのリストを作成しておくとその後柔軟にオペレーションできるので、その前提でスクリプトを作成してみた
#!/bin/zsh

mtx() {
		cmd=`basename $0`
		cat << _EOT_
usege : $cmd [-s <server list file>] [-i] [-w <layout type>]
	-s text file of HOST_NAME in each lines
		if line begins '#', it is comment line.
		default is ./target
	-i each pane accept independent command input. default is NO.
	-w window split layout select : default tiled
		t : tiled
		v : even-vertical
		h : even-horizontal
		e : main-vertical
		o : main-horizontal
	-h show this message
_EOT_
	exit 1
}

INDEPEND=0
LAYOUT="t"
TARGET="./target"

while getopts "s:iw:h" ops
do
	case $ops in
		s)
			TARGET=$OPTARG
			;;
		w)
			LAYOUT=$OPTARG
			;;
		i)
			INDEPEND=1
			;;
		h)
			mtx
			;;
	esac
done

tmux start-server
tmux new-session -d -n tmux-ssh-window -s tmux-ssh

while read line;
do
	echo $line | grep '^#'
	if [ $? -eq 1 ]; then
		tmux split-window -v -t tmux-ssh-window
		tmux send-keys "ssh $line" C-m
		tmux select-layout -t tmux-ssh-window tiled
	fi
done < $TARGET

tmux kill-pane -t 0

tmux select-window -t tmux-ssh-window
tmux select-pane -t 0

if [ $LAYOUT = 't' ]; then
	tmux select-layout -t tmux-ssh-window tiled
elif [ $LAYOUT = 'v' ]; then
	tmux select-layout -t tmux-ssh-window even-vertical
elif [ $LAYOUT = 'h' ]; then
	tmux select-layout -t tmux-ssh-window even-horizontal
elif [ $LAYOUT = 'e' ]; then
	tmux select-layout -t tmux-ssh-window main-vertical
elif [ $LAYOUT = 'o' ]; then
	tmux select-layout -t tmux-ssh-window main-horizontal
fi
if [ $INDEPEND -eq 0 ]; then
	tmux set-window-option synchronize-panes on
fi

tmux attach-session -t tmux-ssh

■ tmuxは便利だが、数百台を一度に相手する場合は厳しい

  • 独自スクリプトをperlで作成してみる
#!/usr/local/bin/perl

use strict;
use warnings;

use File::Basename;
use Getopt::Long 'GetOptions';
use FileHandle;
use IPC::Open2;
use Parallel::ForkManager;

#####MAIN####
if($#ARGV == -1){
	&_usage();
	exit;
}

my $expect_script = '~/bin/mexpect.exp';
my $noexec_ssh = '/usr/bin/ssh -o "StrictHostKeyChecking no" -o "ConnectTimeout 3"';
my $log_dir = '~/.mcom';
my $target_file = './target';
my $noexec;
my $group_num = 256;
my $comm = '';
my $background;
my $retrieve_pass;
my $my_pass = 'hoge';
my $log;
my $server_name = '';
my $errhandl;
my $remainlog;
my $fulllog;

my $user = `whoami`;
chomp($user);

GetOptions('host=s' => \$target_file, 'group=i' => \$group_num, 'noexec' => \$noexec, 'command=s' => \$comm, 'password' => \$retrieve_pass, 'log' => \$log, 'server=s' => \$server_name, 'background' => \$background, 'errhandl' => \$errhandl, 'movenotlog' => \$remainlog, 'fullLog' => \$fulllog);

if(($comm ne '' && $target_file eq '') || ($comm eq '' && !$log)){
	&_usage();
	exit;
}

if(! -d $log_dir){
	system("mkdir -p $log_dir");
	if($? == -1){
		print "ERROR: making log directory failed.\n";
		exit 1;
	}
	elsif($? & 127){
		print "ERROR: making log directory failed.\n";
		exit 1;
	}
	else{
		if($? >> 8 != 0){
			print "ERROR: making log directory failed.\n";
			exit 1;
		}
	}
	system("chmod 777 $log_dir");
}

if($log){
	my $logpath = '';
	$logpath = "$log_dir/.mcom.$user.*.log";

	print "mcom:LOG MODE!\n";
	my $log_exists = `ls $logpath`;
	if($log_exists eq ''){
		exit;
	}
	if($server_name ne ''){
		my $logfile = `grep -l "^spawn ssh .*$server_name" $logpath`;
		if($logfile eq ''){
			exit;
		}
		&_show_mcom_logs($logfile);
	}
	else{
		&_show_mcom_logs($logpath);
	}

	exit;
}
if($comm ne '' && !$remainlog){
	system("rm $log_dir/.mcom.$user.*.log >/dev/null 2>&1");
}

if($comm =~ /\n/){
	print "ERROR: Multiline command cannot be executed.\n";
	exit 1;
}

if($background || $comm =~ / \&$/){
	$comm = &_bg_command($comm);
	$background = 1;
}

open (FILE, $target_file) or die "hostname list cannot open";
my @targets = ();
while(<FILE>){
	chomp $_;
	if($_ ne '' && substr($_, 0, 1) ne '#'){
		push @targets, $_;
	}
}
close(FILE);

if($retrieve_pass){
	$my_pass = &_retrieve_pass;
}

my $manager = new Parallel::ForkManager($group_num);
$manager->run_on_finish(
	sub {
		my ($pid, $exit_code, $ident) = @_;
		if($errhandl && ($exit_code != 0) && !$background && ($group_num > 0)){
			print "mcom:command return code is false. stop fork..\n";
			#$manager->set_max_procs(0);
			$group_num = 0;
		}
	}
);

if($noexec){
	$comm = 'hostname';
}

foreach my $target (@targets){
	my $pid = $manager->start and next;
	if($group_num == 0){
		$manager->finish();
		last;
	}
	my $ret = 0;
	if(!$noexec){
		print "$target:\n";
		$ret = &_async_com($target, $comm);
	}
	else{
		$ret = &_noexec_com($target, $comm);
	}

	if($ret != 0){
		$manager->finish(1);
	}
	else{
		$manager->finish();
	}
}

$manager->wait_all_children;


if($noexec){
	print "\n**************** Server ACCESS results ***********************\n";
	system("cat $log_dir/.mcom.$user.*.log | grep -v spawn | grep -v 'Warning: Permanently added' | grep -v '^Connection to .* closed.'");
}

exit;

sub _usage{
	my $cmd = basename($0, '');
	my $usage_txt = <<"_EOT_";
usage :
	command execution : $cmd -c <command> [-h <hostname list>] [-g <group num>] [-p <server password>] [-b] [-e] [-m] [-n]
		-c : execute Command on remote host on parallel.
		-h : Host list file. default is ./target
		-g : Group, max number of session for remote hosts.
		-p : Password, set password for "sudo" command.
		-b : Background, if command start background process, mcom edit your commandline for right way.
			ex. daemon program
		-e : command Error handling, if command return "error code", stop fork new process.
			in background mode this option not effect.
		-m : reMain log file mode. not delete log before exec command.
		-n : dry ruN mode. try to login hosts in short time and check hostname.
	check execute log : $cmd l [-s <server name> -f] [-u]
		-s : Show log limit by server name
		-f : show Full logfile contents (e.g. spawn ssh../Warning: Permanently../Connection .. closed/)
_EOT_
	print "$usage_txt";
}

sub _show_mcom_logs{
	my $logfile = shift;
	chomp($logfile);
	if($fulllog){
		system("cat $logfile");
	}
	else{
		system("cat $logfile | egrep -v '^FIPS integrity verification|^Warning: Permanently added .* to the list of known hosts\.|^Connection to .* closed\.'");
	}
}

sub _retrieve_pass{
	print "mcom:Enter server password: ";
	system("stty -echo");
	chomp(my $password=<STDIN>);
	print "\n";
	system("stty echo");

	return $password;
}

sub _clean_log{
	system("rm $log_dir/.mcom.$user.*.log");
}

sub _bg_command{
	my $org_comm = shift;
	if($org_comm =~ / \&$/){
		$org_comm =~ s/ \&$//;
		$org_comm = "nohup sh -c \\\"( ( $org_comm &>/dev/null ) & )\\\""
	}
	else{
		$org_comm = "nohup sh -c \\\"( $org_comm &)\\\"";
	}
	return $org_comm;
}

sub _async_com{
	my $host = shift;
	my $comm = shift;
	system("echo '*******************' $host '**************************************' >> $log_dir/.mcom.$user.$$.log");
	my $res = system("$expect_script $host '$comm' $my_pass | tee -a $log_dir/.mcom.$user.$$.log ; exit \${PIPESTATUS[0]}");
	$res = $res >> 8;
	return $res;
}

sub _noexec_com{
	my $host = shift;
	my $comm = shift;
	system("echo '*******************' $host '**************************************' >> $log_dir/.mcom.$user.$$.log");
	my $res = system("$noexec_ssh $host '$comm' | tee -a $log_dir/.mcom.$user.$$.log");
}

1;
  • expectの設定
#!/usr/bin/expect

eval set pass [lindex $argv 2]
eval spawn ssh -t -o \"StrictHostKeyChecking no\" [lindex $argv 0] \"[lindex $argv 1]\"
set timeout -1
while 1 {
	expect {
		-re "Password:.*" {
			send "$pass\r"
			continue
		}
		-re ".sudo. password for .*" {
			send "$pass\r"
			continue
		}
		"does not exist. Create it (y/n)? " {
			send "y\n"
			continue
		}
		eof {
			catch wait result
			set OS_RESULT [ lindex $result 2 ]
			if { $OS_RESULT == -1 } {
					exit 127
			}
			set STATUS [ lindex $result 3 ]
			exit $STATUS
		}
	}
}
⚠️ **GitHub.com Fallback** ⚠️