#! /usr/bin/perl -w
# $Header: /usr/groups/netmaint/boot/RCS/boot-mc.pl,v 1.43 2024/07/17 11:04:39 pb22 Exp $

# Install as www-dyns:/var/www/html/cgi/boot-mc.pl
# scp -p /usr/groups/netmaint/boot/boot-mc.pl www-dyns:/tmp/.
# ssh www-dyns -t sudo cp -v /tmp/boot-mc.pl /var/www/html/cgi/

# Use "pod2text" for info

=head1 NAME

cl-boot-mc - boot a machine

=head1 SYNOPSIS

B<cl-boot-mc>

=head1 OPTIONS

=over 4

=item B<--trace>

Trace the steps being taken, e.g. B<--await> steps

=item B<--quiet>

Be quiet

=item B<--verbose>

Show what it's doing

=item B<--help>

Generate help info.

=item B<--html>

Include HTML tags

=item B<--method> I<method>

Specify the method to use: auto, wol, v20z, xene, amt, wsman, ipmi (default: auto)

=item B<--watch> I<seconds>

Try to watch for any output for up to I<seconds> seconds

=item B<--await> I<ping>|I<ssh>|I<rdp>

Try to await the mahine coming up

=item B<--timeout> I<seconds>

Wait up to at least I<seconds> seconds for the machine to come up (default: method dependent but typically 120 )

=item B<--resume>

Resume, rather than start, a XenE machine

=item B<--server> I<server>

Use the XenE server I<server>

=item B<--broadcast> I<broadcast-address>

Send the WoL packet to IP address I<broadcast-addr>

=item B<--wolcmd|W> I<wolcommand>

Use the command I<wolcommand> (e.g. B<wol>, B<wakeonlan> or B<etherwake>).
Must take "B<-i> I<$broadcast>" to set the broadcast address if needed - not B<etherwake> (for which it is the interface name).
If set to "B<noWOL>" it will use a server to do the WoL.
(default: First available binary in B</usr/bin> of B<wol>, B<wakeonlan> and B<etherwake> (fails if target not on VLAN 100))

=back

=head1 DESCRIPTION

boot a machine

=cut

@dirs	= ( "/usr/groups/netmaint/boot", "/home/pb22/pl", "/var/www/html/cgi");
# TESTING: for these users, use 'admin' - needs HACKs in AA
%admin	= (	"pb22"	=> 1,
		"mas90"	=> 1,
		"jh347"	=> 1,
		"nb110"	=> 1,
		"tb403"	=> 1,
		"mdc56"	=> 1,
		"ppm37"	=> 1,
);

#$wsmanurl='http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_PowerManagementService?SystemCreationClassName=CIM_ComputerSystem,SystemName="Intel(r) AMT",CreationClassName=CIM_PowerManagementService,Name="Intel(r) AMT Power Management Service';
$wsmanurl='http://schemas.dmtf.org/wbem/wscim/1/cim-schema/2/CIM_PowerManagementService';

$priv = $ENV{'SUDO_USER'};
$deftimeout = 120;
$done_wsman = 0;	# Has wsman been tried yet?
$watch = 0;
@wolcmd	= ("wol", "wakeonlan", "etherwake");
$nowolcmd = "noWOL";
sub wolcmd {
	printf STDERR "wolcmd: (%s) @wolcmd\n", $wolcmd||"<UnSet>" if $verbose && ! $quiet;
	return $wolcmd if defined $wolcmd;
	foreach my $cmd (@wolcmd) { return $wolcmd=$cmd if -e "/usr/bin/$cmd" }
	printf STDERR "No wolcmd found\n" if $trace;
	return $wolcmd = $nowolcmd;
}
$onscmd	= "cl-onserver";
$AAconf = "/etc/AMT-AA.conf";
@logfile = ("/var/log/cl_boot-mc_log");

use Getopt::Long;
# abuse bundling_override to allow -W and -w as different
use Getopt::Long qw(:config require_order bundling_override);
use Pod::Usage;
use IO::Socket;

%opts = (
	"await|a:s"	=> \$await,
	"broadcast:s"	=> \$broadcast,
	"help|h"	=> \$help,
	"html"		=> \$html,
	"method|m:s"	=> \$def_method,
	"quiet|q"	=> \$quiet,
	"resume|r"	=> \$resume,
	"server|s:s"	=> \$server,
	"trace|t"	=> \$trace,
	"timeout|ti:s"	=> \$timeout,
	"verbose|v"	=> \$verbose,
	"watch|w:s"	=> \$watch,
	"wolcmd|W:s"	=> \$wolcmd,
);

%methods = (
	"amt"		=> \&boot_mc_amt_wsman,
	"ipmi"		=> \&boot_mc_ipmi,
	"v20z"		=> \&boot_mc_v20z,
	"wol"		=> \&boot_mc_wol,
	"wsman"		=> \&boot_mc_wsman,
	"xene"		=> \&boot_mc_xene,
);

$cmdn = $0;
$cmdn =~ s=^/usr/bin/==;

GetOptions (%opts) || pod2usage();
pod2usage(-verbose => 2) if $help;

$hsc = ($html) ? "<center>" : "";
$hec = ($html) ? "</center>" : "";
$hsem = ($html) ? "<em>".$hsc : "";
$heem = ($html) ? $hec."</em>" : "";
$hsb = ($html) ? "<b>".$hsc : "";
$heb = ($html) ? $hec."</b>" : "";

$hsem, $heem if 0;

$errs = 0;
$comment='';

# Maybe try to find etherwake and wakeonlan?
$ENV{'PATH'} .= ":/sbin:/usr/sbin";

@booted = ();
$start = time;

# Set the user requested default method(s)
undef $def_method if $def_method && $def_method eq "auto";
if ($def_method) {
	@method = split(/[ ,]/, $def_method);
	foreach $one (@method) {
		die("$cmdn: invalid method '$one' - valid methods are: " . join(" ", sort keys %methods) . "\n") unless $methods{$one};
		$def_method{$one} = 1;
	}
}

while ($mc = shift) {
	$mc = lc $mc;
	(my $short = $mc) =~ s/\.[a-z].*//i;
	(my $duff = $mc) =~ s/[a-z0-9_.-]*//ig;
	$errs++;

	# Select the method(s) for this machine - if not user specified, read the 'methods' file.
	$method = $def_method || "";
	%method = %def_method;
	$how = "Invalid characters in machine name: '$duff' - no attempt made";
	if ($duff eq "") {
	    $how = "not booted - email sys-admin\@cl.cam.ac.uk with details for help";
	    if ((! $method) && &open_data_file("methods")) {
		print STDERR "$0: looking for method for $mc\n" if $verbose;
		$found = '';
		while (<DATA>) {
			next if /^#/;
			($host, $val) = m/^(\S+)\s+(.*)/;
			$host = lc $host;
			if ($host eq $mc) {
				print STDERR "# Method $val for $host\n" if $verbose || $trace;
				$found .= ' ' . lc $val;
			}
		}
		if ($found =~ s/^ //) {
			$method = $found;
			@method = split(/[ ,]/, $val);
			%method = (); foreach $one (@method) { $method{$one} = 1; }
			print STDERR "# Method $method for $mc\n" if $verbose || $trace;
		}
		close(DATA);
	    }

	    # Try all the selected methods (or all methods) until one works.
	    foreach $way (sort keys %methods) {
		next if $method && ($method ne $way);	# Skip if this is not one of the requested methods
		print STDERR "# $0: try $way $mc, $short ($method)\n" if $verbose && $trace;
		$cmd =  $methods{$way};
		if ($rc = &$cmd($mc, $short, $method)) {
			$comment = " (comment:$comment)" if $comment && $comment !~ /^\s*$/;
			$how = ($rc == 3) ? 
				"did not need booting using $way$comment" :
				($rc == 2) ? 
				"booted using $way$comment" :
				"boot attempted using $way$comment";
			$errs--;
			push(@booted, $mc);
			last;
		}
	    }
	}
	print $hsb, "$cmdn: $mc $how\n", $heb; # unless $quiet && ! $errs;
	&log("$mc $how") || print ("# Failed to log <<$mc $how>>\n");
}

if ($#booted >= 0 && $await) {
    $timeout = $deftimeout unless defined $timeout;
    foreach $mc (@booted) {
	$secs = time - $start;
        print "$cmdn: $secs: waiting for $mc to be pingable" if $trace && ! $quiet;
        print " [$secs] ping" if $trace &&   $quiet;
	my ($upto) = time + $timeout;
	while ($upto > time) {
		$! = 0;
		last unless &mysystem("ping -c 1 $mc > /dev/null");
		print "." if $trace;
		sleep(1);	# just in case command returns immediately
	}
	print "\n" if $trace && ! $quiet;
	if ($upto <= time) {
		$secs = time - $start;
		print "\n$cmdn: $secs: $mc not responding within $timeout s\n";
		$errs ++;
    	}
	if ($errs) {
	} elsif ($await eq "ping") {
		$secs = time - $start;
		print "$cmdn: $secs: $mc pingable\n" if $trace && ! $quiet;
		print " [$secs] $mc pingable\n" if $trace &&   $quiet;
	} elsif ($await eq "ssh") {
		$secs = time - $start;
		print "$cmdn: $secs: $mc pingable so try ssh\n" if $trace && ! $quiet;
		print " [$secs] ssh" if $trace && ! $quiet;
		if (&try_ssh($mc, $upto)) {
			$secs = time - $start;
			print "$cmdn: $secs: $mc accessible via ssh\n" if $trace && ! $quiet;
			print " [$secs] $mc sshable\n" if $trace &&   $quiet;
		} else {
			$secs = time - $start;
			print "\n$cmdn: $secs: $mc not responding to ssh within $timeout s\n";
			$errs ++;
		}
	} elsif ($await eq "rdp") {
		$secs = time - $start;
		print "$cmdn: $secs: $mc pingable so try rdp\n" if $trace && ! $quiet;
		print " [$secs] rdp" if $trace && ! $quiet;
		if (&try_rdp($mc, $upto)) {
			$secs = time - $start;
			print "$cmdn: $secs: $mc accessible via rdp\n" if $trace && ! $quiet;
			print " [$secs] $mc rdpable\n" if $trace &&   $quiet;
		} else {
			$secs = time - $start;
			print "\n$cmdn: $secs: $mc not responding to rdp within $timeout s\n";
			$errs ++;
		}
	} else {
		$secs = time - $start;
		print "$cmdn: $secs: $mc pingable (unrecognised await: '$await')\n" if $trace && ! $quiet;
		print " [$secs] $mc pingable [$await]\n" if $trace &&   $quiet;
	}
    }
}

exit ($errs);

# =============================================================================

# Boot a machine using Wake On Lan - need to have per host data
sub boot_mc_wol {
	my ($mc, $short) = @_;
	my ($done) = 0;
	my ($file) = "boot_wol_file";
	&bmc_off($mc, $short);
	unless (&open_data_file($file)) { warn("$cmdn: failed to open $file: $!\n"); return 0; }
	while(<DATA>) {
		next if /^#/;
		next if /^\s*$/;
		chomp;
		$thiscomment = '';
		@fields = split(/\s/, $_);
		## print STDERR ": $mc: $fields[0]:  $_\n" if $verbose;
		next unless $fields[0] =~ /^$mc$/i || $fields[0] =~ /^$short$/i;
		## print STDERR ": $mc: |$fields[0]|$fields[1]|$fields[2]|  $_\n" if $verbose;
		$thiscomment = $1 if /\scomment="([^"]+)"/;
		$cmd = &wolcmd;
		$qcmd = $cmd;
		$cmd .= " -v" if $trace || $verbose;
		$cmd .= " > /dev/null" if $quiet;
		if ($broadcast) { $cmd .= " -i $broadcast" }
		elsif ($fields[2] && $fields[2] =~ /^\d+\./) { $cmd .= " -i $fields[2]" }
		$cmd .= " $fields[1]";
		$qcmd .= " $fields[1]";
		if ($cmd =~ / -i / && ! &multipleinterfaces) {
			print STDERR "Need to run on another machine: $cmd\n" unless $quiet;
			$qcmd = "remote $qcmd";
			$cmd =~ s/^\S+/cl-onserver --wol -- /;
		} elsif ($nowol || $wolcmd eq $nowolcmd) {
			$qcmd = "remote $qcmd";
			$cmd =~ s/^\S+/cl-onserver --wol -- /;
		}
		print STDERR "$cmdn: run: $cmd\n" if ($verbose || $trace) && ! $quiet;
		print STDERR "$cmdn: $qcmd" if ($verbose || $trace) &&   $quiet;
		&mysystem($cmd);
		if ($?) { warn("$cmdn: '$cmd' gave $? '$!'\n"); }
		else { $done++; }
		$comment .= " $thiscomment";
	}
	close(DATA) ||  warn("$cmdn: failed to close $file: $!\n");

	# HACK to allow MAC address without ":"s
	if (!$done && $mc =~ /^[a-fA-F0-9]{12}$/) {
		 $mc =~ s/(..)/$1:/g;
		 $mc =~ s/:$//;
	}
	# Not booted, and it looks like a MAC address
	if (!$done && $mc =~ /^..:..:..:..:..:..$/) {
		print STDERR ": Using user supplied MAC address: $mc\n" if $verbose;
		$cmd = &wolcmd;
		if ($broadcast) { $cmd .= " -i $broadcast" }
		$cmd .= " $mc";
		if ($cmd =~ / -i / && ! &multipleinterfaces) {
			print STDERR "Need to run on another machine: $cmd\n";
			$cmd =~ s/^\S+/cl-onserver --wol -- /;
		} elsif ($nowol || $wolcmd eq $nowolcmd) {
			$cmd =~ s/^\S+/cl-onserver --wol -- /;
		}
		print STDERR "$cmdn: run: $cmd\n" if ($verbose || $trace) && ! $quiet;
		print STDERR "$cmdn: $cmd" if ($verbose || $trace) &&   $quiet;
		&mysystem($cmd);
		if ($?) { warn("$cmdn: '$cmd' gave $? '$!'\n"); }
		else { $done++; }
	}
	return $done;
}

# -----------------------------------------------------------------------------

# Boot a machine using ipmi - no data needed, but can use '.bmc' as heuristic
sub boot_mc_ipmi {
	return unless $method;
	my ($mc, $short) = @_;
	my ($name) = &bmc_name($mc, $short);
	my ($op) = $ENV{"IPMIOP"} || "on";
	my ($user, $pw, $pwfile) = &user_pw_file('ipmi-');
	$pw if 0; # we happen not to need $pw ...

	unless ($user) {
		print STDERR "# $0: IPMI userid unknown for $mc ($short)\n" if $way;
		return 0;
	}

	# PW can be provided:
	# -f $file	read PW from file
	# -a		prompt for PW (default)
	# -E		PW is in the env var IPMI_PASSWORD
	# -P $pw	PW is $pw (tries to hide it)
	my ($args) = (-r $pwfile) ? " -f $pwfile < /dev/null" : "";

	# Check that the name is known
	unless ($name) {
		print STDERR "# $0: IPMI boot for unknown $mc ($short)\n" if $way;
		return 0;
	}
	print STDERR "# $0: IPMI boot $mc ($short) $name (ipmitool -I lanplus -H $name -U $user$args)\n" if $verbose;

	# See if it's already running
	chomp($state = `ipmitool -I lanplus -H $name -U $user$args power status`);
	$state =~ s/^\n*\s*//m;
	print STDERR "# Gave '$!' '$?' '$state'\n" if $verbose;
	# 'Chassis Power is on'
	if ($state =~ /Chassis Power is $op/) {
		print STDERR "# $0: already running: $state\n";
		$deftimeout = $deftimeout / 2;
		return 3;
	}
	if ($state =~ /Bad hostname/) {
		print STDERR "# $0: $state\n";
		return 0;
	}
	if ($state =~ / Connection refused/) {
		print STDERR "# $0: Not an IPMI system? : '$state'\n" if $way;
		return 0;
	}

	# Wake it up ....
	print STDERR "# $0: IPMI powerup $mc ($short) $name\n" unless $quiet;
	chomp($state = `ipmitool -I lanplus -H $name -U $user$args power $op < /dev/null 2>&1`);
	$state =~ s/^\n*\s*//m;
	print STDERR "# Gave", ($?) ? " '$!' '$?'" : "", " '$state' - see if up (may take a while)\n" unless $quiet;

	# Give it a chance to start:
	sleep(1);

	# Awake?
	chomp($state = `ipmitool -I lanplus -H $name -U $user$args power status < /dev/null`);
	print STDERR "# Gave '$!' '$?' '$state'\n" if $verbose;
	if ($state =~ /Chassis Power is $op/) {
		print STDERR "# $0: running: $state\n";
		$deftimeout = $deftimeout * 2;
		return 2;
	}

	# Give it a chance to start:
	sleep(5);

	# Try again ...
	chomp($state = `ipmitool -I lanplus -H $name -U $user$args power status < /dev/null`);
	print STDERR "# Gave '$!' '$?' '$state'\n" if $verbose;
	if ($state =~ /Chassis Power is $op/) {
		print STDERR "# $0: running: $state\n";
		$deftimeout = $deftimeout * 2;
		return 2;
	}
	print STDERR "# $0: gave: '$state'\n";
	return 0;
}

# -----------------------------------------------------------------------------

# Boot a machine using Sunfire v20z ssh "poweron" HACK - legacy method, so assume all are enumerated in 'methods'
sub boot_mc_v20z {
	my ($mc, $short) = @_;
	(my $bmc = $mc) =~ s/^([^.]+)/$1.bmc/;
	my ($on) = 0;
	return 0 unless $mc =~ /^pb0(\d\d)$/ || $mc =~ /^pb0(\d\d)\./;
	return 0 unless 0 <= $1 && $1 <= 8;
	#return 0 if $1 == 9;	# pb009 isn't there

	# unless (open(CMD, "ssh -ntt poweron\@$bmc < /dev/null |")) { warn("$cmdn: ssh poweron\@$bmc failed: $!\n"); return 0; }
	unless ($child = open(CMD, "-|")) {
		open(STDERR, ">&STDOUT");
		close(STDIN);
		exec "ssh", "-ntt", "poweron\@$bmc";
		die("$cmdn: ssh poweron\@$bmc failed: $!\n");
	}
	while (<CMD>) {
		next if /^Sun Microsystems\r$/;
		next if /^IPMI v2.0 Service Processor\r$/;
		next if /^IPMI v1.5 Service Processor\r$/;
		next if /^Version:  V2.4.0.14\r$/;
		next if /^Version:  V2.1.0.16\r$/;
		print unless /^$/ || /^\r$/;
		# print ":# $_";
		$on = 1 if /system is already on/ || /system is On/;
		if ($on && ! $watch) {
			kill 2, $child;
			sleep 0.1;
			kill 9, $child;
			return 1;
		}
	}
	unless (close(CMD)) { warn("$cmdn: ssh poweron\@$bmc failed: $?\n"); return 0; }

	return 0 if $on;
	return 1;
}

# -----------------------------------------------------------------------------
# Get a user and PW
sub user_pw_file {
	my ($type) = @_;
	my $user = '';
	my $pw = '';
	for $dir ($ENV{'HOME'}, "/root") {
		next unless $dir;
		next unless -r "$dir/.${type}user" && -r "$dir/.${type}user";
		$user = `cat $dir///.${type}user`;
		$pw = `cat $dir///.${type}pw`;
		$pwfile = "$dir///.${type}pw", last if $pw && $user;
	}
	if ((! $user || ! $pw) && $type eq "amt" && -e $AAconf && ($user = $ENV{'AAPRINCIPAL'})) {
		print STDERR "# $0: Look for $user in $AAconf\n" if $verbose;
		$user = "admin" if $admin{$user};
		print STDERR "# $0: Look for $user in $AAconf\n" if $verbose;
		if (open(CONF, $AAconf)) {
                	while (<CONF>) { $pw = $1 if /^$user\s+(\S+)\s*$/; }
			close(CONF);
			print STDERR "# $0: found PW for $user in $AAconf\n" if $pw && ! $quiet;
		}
	}
	chomp($pw) if $pw;
	chomp($user) if $user;
	die("$0: Failed to find user $type credentials\n") unless $pw && $user;
	die("$0: Invalid user '$user'\n") unless $user =~ /^[a-z]*[0-9]*$/;
	die("$0: Invalid password\n") unless $pw =~ /^[a-zA-Z0-9\!\@\#\$\%\^\&\*\(\)]*$/;
	return ($user, $pw, $pwfile);
}

# -----------------------------------------------------------------------------

# Boot a machine using an AMT BMC using wsman - no data needed, but can use '-bmc' as heuristic
sub boot_mc_wsman {
	return 0 if ($done_wsman++);

	my($mc, $short, $way) = @_;
	my($name) = &bmc_name($mc, $short);
	my($state);
	my($wsmanop) = $ENV{'WSMANOP'} || "on";
	my($xmld1) = "/global/src/etc/wsman";
	my($xmld2) = "/etc/wsman";
	my($xmld) = (-e $xmld2) ? $xmld2 : $xmld1;
	my($xml) = "$xmld/amt-$wsmanop.xml";
	my ($user, $pw) = &user_pw_file('amt');
	$way = 'amt/wsman' if $way eq 'amt';
	unless ($user) {
		print STDERR "# $0: cannot use wsman as no credentials\n";
		return 0;
	}
	my $R = $ENV{'R'} ? " -R" : "";
	my $fmt = "wsman invoke -a RequestPowerStateChange$R %s $wsmanurl -P 16992 -h $name -u $user -p '%s' -V -v";
	$wsmanop = "on" if $priv;

	# Check that the name is known
	unless ($name) {
		print STDERR "# $0: wsman AMT boot for unknown $mc ($short)\n" if $way;
		return 0;
	}
	my $cmd = sprintf($fmt, "", $pw);
	my $deb = sprintf($fmt, "", '<pw>');
	print STDERR "# $0: wsman poll $mc ($short) $name for $user using '$deb'\n" if $verbose;

	chomp($state = `$cmd </dev/null 2>&1`);
	$stateH = $state;
	if ($html) { $stateH =~ s/</\&lt;/g; $stateH =~ s/>/\&gt;/g; }
	print STDERR "state ($deb) = <<< $stateH >>>\n" if $verbose;
	if ($state =~ /<g:ReturnValue>(\d+)<\/g:ReturnValue>/) {
		my ($rc) = $1;
		if ($rc eq "2") {
			print STDERR "# $0: as 'expected' poll returned $rc\n" unless $quiet;
			# $deftimeout = $deftimeout / 2;
			# return 3;
		} else {
			print "# existing state $1\n";
		}
	} else {
		print "# existing <<< $state >>> not recognised\n";
	}

	$cmd = sprintf($fmt, "-J $xml", $pw);
	$deb = sprintf($fmt, "-J $xml", '<pw>');
	print STDERR "# $0: wsman boot $mc ($short) $name for $user using '$deb' iffy as no '$xml'\n" unless -r $xml;
	print STDERR "# $0: wsman boot $mc ($short) $name for $user using '$deb'\n" if $verbose;
	chomp($state = `$cmd < /dev/null 2>&1`);
	$stateH = $state;
	if ($html) { $stateH =~ s/</\&lt;/g; $stateH =~ s/>/\&gt;/g; }
	print STDERR "state ($deb) = <<< $stateH >>>\n" if $verbose;
	if ($state =~ /<g:ReturnValue>(\d+)<\/g:ReturnValue>/) {
		my ($rc) = $1;
		if ($rc eq "0") {
			print STDERR "# $0: now $wsmanop: $rc\n" unless $quiet;
			return 2;
		}
		print "# change state gave RC $1\n";
	} else {
		print "# new <<< $state >>> not recognised\n";
	}
	return 0;
}

# -----------------------------------------------------------------------------

# Boot a machine using an AMT BMC using amttool (or wsman if it fails 'FAULT: 404 Not Found' - no data needed, but can use '-bmc' as heuristic
sub boot_mc_amt_wsman {
	my ($mc, $short, $way) = @_;
	my($name) = &bmc_name($mc, $short);
	my($state);
	my ($try) = 0;

	# Check that the name is known
	unless ($name) {
		print STDERR "# $0: SOAP / wsman AMT boot for unknown $mc ($short)\n" if $way;
		return 0;
	}
	print STDERR "# $0: SOAP / wsman AMT boot $mc ($short) $name\n" if $verbose;

	# See if it's already running
	$ENV{'AMTTO'} = 4;
	$ENV{'AMT_DEBUG'} = ($verbose) ? 1 : 0;
	$ENV{'AAPRINCIPAL'} = 'admin' if $ENV{'AAPRINCIPAL'} && $admin{$ENV{'AAPRINCIPAL'}};
	chomp($state = `cl-amttool $name state < /dev/null 2>&1`);
	$state =~ s/^\n*\s*//m;
	$state =~ s/\n\s*/; /gm;
	my($txt) = ($?) ? "'$!' " : "";
	print STDERR "# $0: 'state' gave $txt'$?' '$state'\n" if $verbose && $?;
	# new iAMT: 'state' gave '' '256' 'FAULT: 404 Not Found'
	return &boot_mc_wsman(@_) if ($state =~ 'FAULT: 404 Not Found' || $state =~ /no AMT PW found/);
	print STDERR "# $0: 'state' gave $txt'$?' '$state'\n" if $verbose && ! $?;
	if ($state =~ / Caught ALARM/) {
		print STDERR "# $0: Not an AMT system? : '$state'\n" if $way && $verbose;
		$ENV{'AMTTO'} = 2;
		chomp($state = `cl-amttool $name state < /dev/null`);
		$state =~ s/^\n*\s*//m;
		print STDERR "# 'state' gave '$!' '$?' '$state'\n" if $verbose;
	}

	if ($state =~ /Powerstate.*S0/) {
		print STDERR "# $0: already running: $state\n" if $verbose;
		$deftimeout = $deftimeout / 2;
		return 3;
	}
	if ($state =~ /Bad hostname/) {
		print STDERR "# $0: $state\n";
		return 0;
	}
	if ($state =~ / Connection refused/) {
		print STDERR "# $0: Not an AMT system? : '$state'\n" if $way;
		return 0;
	}
	if ($state =~ /no AMT PW/) {
		print STDERR "# $0: No password found : '$state'\n" if $way;
		return 0;
	}
	if ($state =~ / Caught ALARM/) {
		print STDERR "# $0: Not an AMT system? : '$state'\n" if $way;
		return 0;
	}
	if ($state =~ /FAULT: 401 Unauthorized/) {
		print STDERR "# $0: no valid credentials : '$state'\n" if $way;
		return 0;
	}

	# Wake it up ....
	print STDERR "# $0: AMT powerup $mc ($short) $name\n" unless $quiet;
	chomp($state = `echo y | cl-amttool $name powerup 2>&1`);
	$state =~ s/^\n*\s*//m;
	# 'host acs-01-bmc: powerup  [y/N] ? done.'
	$state =~ s/\s+\[y\/N\] \?\s+/ /;
	print STDERR "# 'powerup' gave", ($?) ? " '$!' '$?'" : "", " '$state' - see if up (may take a while)\n" unless $quiet;

	foreach $try (1 .. 5) {
		chomp($state = `cl-amttool $name state < /dev/null`);
		print STDERR "# try $try: 'state' gave '$!' '$?' '$state'\n" if $verbose;
		if ($state =~ /Powerstate.*S0/) { print STDERR "# $0: on try $try: running: $state\n"; return 2; }
	}

	print STDERR "# $0: gave: '$state' after $try tries\n";
	return 0;
}

# -----------------------------------------------------------------------------

# Boot a Xen Enterprise VM  - no data needed - no heuristic
sub boot_mc_xene {
	return unless $method;
	my ($mc) = @_;
	my ($done) = 0;

	$done++ if &call_on_server($mc, $server);
	$deftimeout = $deftimeout * 2 if $done;
	return $done;
}

sub call_on_server {
	my ($mc, $servers) = @_;
	$servers =~ s/,/ /g if $servers;
	$cmd = $onscmd;
	$cmd .= " --server '$servers'" if $servers && ($servers =~ /../);
	$cmd .= ($resume) ? " --xe vm-resume" : " --xe vm-start";
	$cmd .= " vm=$mc";
	print STDERR "$cmdn: run: $cmd\n" if ($verbose || $trace) && ! $quiet;
	print STDERR "$cmdn: $cmd" if ($verbose || $trace) &&   $quiet;
	# system($cmd);
	# if ($?) { warn("$cmdn: '$cmd' gave $? '$!'\n"); return 0; }
	open(CMD, "$cmd 2>&1|") || die("$cmdn: failed to run '$cmd': $!\n");
	$p_novm = $p_running = '';
	while (<CMD>) {
		# Already running -- time to exit
		# Build up to "already running"
		# You attempted an operation on a VM that was not in an appropriate power state at the time; for example, you attempted to start a VM that was already running.  The parameters returned are the VM's handle, and the expected and actual VM state at the time of the call.
		# vm: e8746195-3c06-c1d9-c1ca-e46ddfd16c03 (ssh-remote-0)
		# expected: halted
		# actual: running
		# /bin/cl-xene-run-cmd: 'xe vm-start vm=ssh-remote-0' using alarm(2) gave 256
		# # exit code 1 for sxp21 / sxp21: /usr/bin/cl-onserver --server sxp21 sxp09 sxp14 sxp20 sxp00 bretzel --xe vm-start vm=ssh-remote-0
		if (/^You attempted an operation on a VM that was not in an appropriate power state at the time; / ||
		    /^vm: / || /^expected: halted/) { $p_running .= "= $_"; next; }
		if (/^actual: running/) { print "$cmdn: machine already running\n" unless $quiet; last; }
		print "$p_running"; $p_running = '';

		# Called the wrong server
		# Error: No matching VMs found
		# /bin/cl-xene-run-cmd: 'xe vm-start vm=ivatt' using alarm(2) gave 256
		# exit code 1 for sxp00 / sxp00: /usr/bin/cl-onserver --server sxp00 sxp09 sxp14 sxp20 sxp21 bretzel --xe vm-start vm=ivatt
		if (/^Error: No matching VMs found/|| /\/bin\/cl-xene-run-cmd: 'xe vm-start/) { $p_novm .= "@ $_"; next; }
		if (/^# exit code 1 for/) { $p_novm = ''; next; }
		print "$p_novm"; $p_novm = '';

		print ": $_";
	}
	close(CMD);
	return 1;
}

# -----------------------------------------------------------------------------

# find a BMC's name
sub bmc_name {
	my ($mc, $short) = @_;
	return $bmc_name if defined($bmc_name);
	return $mc if $mc =~ /^[0-9.]+$/;
	if (($mc =~ /^[^.]*bmc($|\.)/) && gethostbyname($bmc_name = $mc)) { ; }
	elsif (gethostbyname($bmc_name = "$mc-bmc")) { ; }
	elsif (gethostbyname($bmc_name = "$mc.bmc")) { ; }
	elsif (gethostbyname($bmc_name = "$mc-bmc.cl.cam.ac.uk")) { ; }
	elsif (gethostbyname($bmc_name = "$mc.bmc.cl.cam.ac.uk")) { ; }
	elsif (gethostbyname($bmc_name = "$short-bmc")) { ; }
	elsif (gethostbyname($bmc_name = "$short.bmc")) { ; }
	elsif (gethostbyname($bmc_name = "$short-bmc.cl.cam.ac.uk")) { ; }
	elsif (gethostbyname($bmc_name = "$short.bmc.cl.cam.ac.uk")) { ; }
	else { $bmc_name = ''; }
	return $bmc_name;
}

# -----------------------------------------------------------------------------

# see if a BMC is unavailable
sub bmc_off {
	my ($mc, $short) = @_;
	my($name) = &bmc_name($mc, $short);
	return 0 unless $name;
	$err = `ping -c 1 $name 2>&1 > /dev/null`;
	if (! $?) {
		warn("$cmdn: $mc\'s BMC ($name) is accessible\n") unless $quiet
	} elsif ($err =~ /icmp open socket: Permission denied/) {
		warn("$cmdn: $mc\'s BMC ($name) may be accessible [SELinux feature]\n") unless $quiet
	} else {
		chomp $err;
		warn("$cmdn: $mc\'s BMC ($name) is not accessible ($err)\n")
	}
}

# -----------------------------------------------------------------------------

# open a data file boot_wol_file
sub open_data_file {
	my($file) = @_;
	foreach $dir (@dirs) { return DATA if open(DATA, "$dir/$file") }
	return 0;
}

# -----------------------------------------------------------------------------

# check if this machine has multiple interfaces
sub multipleinterfaces {
	if (open(INT, "/proc/net/dev")) {
		my ($interfaces) = 0;
		while (<INT>) {
			$interfaces++ if /^\s*\S+:\d+/;
		}
		close(INT);
		return $interfaces > 3;
	}
	print STDERR "# $cmdn: failed to detech NIC count: $!\n" unless $! eq "Permission denied";
	return 0;
}


sub try_ssh {
	my ($mc, $upto) = @_;
	while (time < $upto) {
		my ($res);
		sleep(1);
		next unless $sock = IO::Socket::INET->new(PeerAddr => $mc, PeerPort => 22, Proto => 'tcp', Blocking => 1);
		$len = sysread($sock, $res, 4);
		next unless defined $len && $len > 0;
		return 1;
	}
	return 0;
}

sub try_rdp {
	my ($mc, $upto) = @_;
	while (time < $upto) {
		my ($res);
		sleep(1);
		next unless $sock = IO::Socket::INET->new(PeerAddr => $mc, PeerPort => 3389, Proto => 'tcp', Blocking => 1);
		# print STDERR "Got socket ...";
		my ($startt) = time;
		$len = send($sock, "OK\r\n", 0);
		# print STDERR "write len=$len\n";
		$len = sysread($sock, $res, 4);
		my ($endt) = time;
		my ($duration) = $endt - $startt;
		# printf STDERR "read duration=$duration, len=%s in %d ms / %d is ($startt vs $endt)\n", $len || "<UnSet>", ($endt - $startt) * 1000, ($endt - $startt);
		next if $duration > 1;
		#$len = syswrite($sock, "\r\n\r\n", 4);
		#print STDERR "write len=$len\n";
		#$len = sysread($sock, $res, 4);
		#print STDERR "read len=$len\n";
		#next unless defined $len && $len > 0;
		return 1;
	}
	return 0;
}

# cl.cam.ac.uk: use data from dot files.
sub file_val {
	my ($file) = @_;
	my @files = ( "/etc/$file" );
	unshift @files, "$ENV{'HOME'}/$file" if $ENV{'HOME'};

	foreach my $file (@files) {
		if (open(PWFILE, $file)) {
			my $val = <PWFILE>;
			close(PWFILE);
			chomp $val;
			return $val if $val;
		}
	}
	return undef;
}

sub mysystem {
	my ($arg) = @_;
	print STDERR "# system($arg)\n" if $verbose;
	return system($arg);
}

# -----------------------------------------------------------------------------

# log a message
sub log {
    for $logfile (@logfile) {
	if (open(OUT, ">>$logfile")) { print OUT "" . localtime, " $0 ", $ENV{'REMOTE_USER'} || ("l-". $ENV{'USER'}), ": ",  @_, "\n"; close(OUT); return 1; }
	else { print "# $0: Failed to log to $logfile: @_: $!\n"; }
    }
    return 0;
}
