#! /usr/bin/perl -w
# $Header: /global/src/etc/cl-manage-scripts/RCS/cl-update-system,v 1.41 2023/07/17 07:08:20 pb22 Exp $
# Use "pod2text" for info

=head1 NAME

cl-update-system - do a standard potted update of the system, keeping things quiet

=head1 SYNOPSIS

cl-update-system

=head1 OPTIONS

=over 4

=item B<--verbose|v>

generate comments as to what it is doing.

=item B<--trace|t>

trace things as they happen, rather than just showing output if there is anything interesting.

=item B<--help|h>

generate help info.

=item B<--ownfiles|o>

call B<cl-make-ownfiles> to generate system info.

=item B<--elmer|e>

Force "elmer is accessible" flag, used to decide whether to note status by touching a file on elmer

=back

=head1 DESCRIPTION

Run commands up update the system by updating existing RPMs, adding new RPMs and applying any configurations

=cut

use Getopt::Long;
use Pod::Usage;
use Text::Wrap;

$Text::Wrap::columns=900;

%opts = (
	"verbose|v"	=> \$verbose,
	"help|h"	=> \$help,
	"elmer|e"	=> \$elmer,
	"trace|t"	=> \$trace,
	"filter|f"	=> \$filter,
	"odnfiles|o"	=> \$ownfiles,
);


GetOptions (%opts) || pod2usage();
pod2usage(-verbose => 2) if $help;

%skip = (
	"Building dependency tree..."	=> 1,
	"Building Dependency Tree..."	=> 1,	# sarge
	"Preconfiguring packages ..."	=> 1,
	"Reading package lists..."	=> 1,
	"Reading Package Lists..."	=> 1,	# sarge
	"Reading state information..."	=> 1,
	""				=> 1,
);

%boutskip = (
	''								=> 1,
	'Loading "fastestmirror" plugin'				=> 1,
	'Loading "installonlyn" plugin'					=> 1,
	'Setting up Update Process'					=> 1,
	'Setting up repositories'					=> 1,
	'Loading mirror speeds from cached hostfile'			=> 1,
	'Reading repository metadata in from local files'		=> 1,
	'Excluding Packages in global exclude list'			=> 1,
	'Downloading DeltaRPMs:'					=> 1,
	'Rebuilding rpms from deltarpms'				=> 1,
	'Running rpm_check_debug'					=> 1,
	'Finished'							=> 1,
	'Determining fastest mirrors'					=> 1,
	'Resolving Dependencies'					=> 1,
	'--> Populating transaction set with selected packages. Please wait.' => 1,
	'--> Running transaction check'					=> 1,
	'--> Finished Dependency Resolution'				=> 1,
	'Dependencies Resolved'						=> 1,
	'Downloading Packages:'						=> 1,
	'Running Transaction Test'					=> 1,
	'Finished Transaction Test'					=> 1,
	'Transaction Test Succeeded'					=> 1,
	'Running Transaction'						=> 1,
	'Loading mirror speeds from cached hostfile'			=> 1,
	'Setting up and reading Presto delta metadata'			=> 1,
);

$reportdir = "/anfs/bigtmp/.report-update";
$changelog = "/etc/yum/pluginconf.d/changelog.conf";

$cus = $0;
$cus =~ s=/usr/bin/==;
$cus =~ s=/global/src/etc/manage-scripts=gs=;

# Are extra privs needed?
$priv = ($< == 0) ? "" : "cl-asuser ";

# Can we 
$elmer = &can_touch_elmer() unless defined($elmer);

$exit = 0;
$repos = $basics = $details = $full_log = "";

$ENV{'DEBIAN_FRONTEND'} = "noninteractive" unless $trace;

process_apt_get("-"), exit if $filter;

# Do the business.
&do_update();
# Do not add RPMs and apply patches (yet) on FC 1-3
&do_add();
&run_cmd("cl-patch-file -a");
&run_cmd("cl-hostid-fix --update");
&do_ownfiles() if $ownfiles;
&report_status();
&log($details, $details);
unless ($priv) {
	system('find /var/log/. -name cl-update\* -mmin +1000 -exec rm {} \;');
	$version = time;
	if (open(LOG, ">>/var/log/cl-update-$version")) {
		print LOG "# $cus: final exit $exit\n$basics\n\n$full_log";
		close(LOG);
	}
}
exit($exit);


# -----------------------------------------------------------------------------

# Determine whether elmer is accessible
# Should do it with a protective timeout in a sub process, etc ....
sub can_touch_elmer {
	return 0 if $< && $>;
	return 1 if -w "$reportdir/.";
	return 0;
}

# -----------------------------------------------------------------------------

# Report the status of the job
sub report_status {
	&log($verbose || $trace, "# $cus: report status ($elmer)\n");

	# If there i a good chance of reporting it by touching a file, do so.
	if ($elmer) {
		$host = `hostname`;
		chomp $host;
		$host =~ s/.cl.cam.ac.uk//;
		$file = "$reportdir/$host";
		if (open(TMP, ">> $file")) { utime time, time, $file; close(TMP) }
		else { print STDERR "# $cus: $file gave $!\n"; }
	}
	# Otherwise, if root, force some text to make cron send email
	else {	&log(!$verbose && !$trace && !$< && !$>,
			"# $cus completed at " . localtime() . "\n");
	}
}

# -----------------------------------------------------------------------------


sub do_update {
	my($yum) = -e "/usr/bin/yum";
	my($apt) = -e "/usr/bin/apt-get";

	# if just one found, call it. otherwise, try both
	if ($yum && ! $apt) { &do_yum_update; }
	elsif (!$yum && $apt) { &do_apt_get_update; }
	else {
		&do_yum_update;
		&do_apt_get_update;
	}
}

sub do_apt_get_update {
	# apt-get generates: get, prepare, unpack, setup
	# Get: 23 http://gb.archive.ubuntu.com hardy-updates/main totem-gstreamer 2.22.1-0ubuntu3 [777kB]
	# Preparing to replace totem-gstreamer 2.22.1-0ubuntu2 (using .../totem-gstreamer_2.22.1-0ubuntu3_amd64.deb) ...
	# Unpacking replacement totem-gstreamer ...
	# Setting up totem-gstreamer (2.22.1-0ubuntu3) ...

	$quiet = ($trace) ? "" : "-q";
	# use the installer's setting -- we post patch
	$conf = ($trace) ? "" : "-o DPkg::Options::=--force-confnew";
	&do_apt_get("${priv}apt-get $quiet update 2>&1", "primecahe");
	&do_apt_get("${priv}apt-get -y -f $conf upgrade 2>&1", "upgrade");
	&do_apt_get("${priv}apt-get -y $quiet clean 2>&1", "clean");
	my($aptgethelp) = `apt-get -h | tr '\12' ';'`;
	&do_apt_get("${priv}apt-get -y $quiet autoremove 2>&1", "autoremove") if $aptgethelp =~ /autoremove/;
}

sub do_apt_get {
	my ($cmd, $op) = @_;
	$boring = 0;
	$complete = '';
	$seen{""} = $seenG{""} = $seenU{""} = "";
	$hit = "";
	$got = $todo = 0;

	# Run "apt-get" and collect what we get
	&log($verbose, "# $cus: call: $cmd\n");
	$start = time;
	if ($trace) {
		$rc = system($cmd);
		warn("# $cus: $cmd' failed: $rc\n") if $rc;
	} else {
		open(STDIN, $cmd."|") || die("$cus: command '$cmd' failed: $!\n");
		&process_apt_get($cmd, $op);
	}

	# Decide whether to output anything
	my($dolog) = $verbose || $trace || $boring <= 0;
	&log($dolog, "# $cus: '$cmd' ", time - $start, "s $rc $boring$complete:\n");
	(($dolog) ? $details : $basics) .= "-" x 30 . sprintf(" %-10s ", $op) . "-" x 30 . "\n$out\n" if $out;
}

sub process_apt_get {
	my ($cmd, $op) = @_;
	my ($bout) = "";
	my ($conf) = "";
	my ($conferrs) = "";
	my ($conferr) = 0;
	my ($debconf) = "";
	my ($mods) = "";
	my ($extras) = "";
	my (%repos) = ( '' => 0 );
	while (<>) {
		s/\15$//;
		$out .= $_;
		$fl = $_;
		chomp;
		if ($skip{$_}) {;}

		# Hit http://gb.archive.ubuntu.com maverick/main amd64 Packages
		elsif (/^(Hit|Ign)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)$/) { $hit .= " $1:$2:$3:$5:$4"; $repos{$2}++ }
		# Hit http://gb.archive.ubuntu.com intrepid-updates/main Packages
		elsif (/^(Hit|Ign) (\S+) (\S+) (\S+)$/) { $hit .= " $1:$2:$3:$4"; $repos{$2}++ }
		# Hit:2 http://www-uxsup.csx.cam.ac.uk/pub/linux/ubuntu xenial InRelease
		elsif (/^(Hit|Ign)(:\d+)\s+(\S+)\s+(\S+)\s+(\S+)$/) { $hit .= " $1:$2:$3:$4:$5"; $repos{$3}++ }
		elsif (/^(Hit|Ign)(:\d+)\s+(\S+)\s+(\S+)\s+(\S+)\s+(\S+)$/) { $hit .= " $1:$2:$3:$4:$6:$5"; $repos{$3}++ }
		# Ign http://www.mendeley.com  Release
		elsif (/^(Hit|Ign) (\S+) (\S*) (\S+)$/) { $hit .= " $1:$2:$3:$4"; $repos{$2}++ }

		# 0 upgraded, 0 newly installed, 0 to remove and 1 not upgraded.
		elsif (/^(\d+) upgraded, (\d+) newly installed, (\d+) to remove and (\d+) not upgraded\.$/) {
			$mods .= " U=$1 N=$2 R=$3 NU=$4";
			$todo += $1 + $2;
		}

		# Err http://packages.medibuntu.org hardy Release
		# Get:1 http://www.cl.cam.ac.uk CL Release.gpg [189B]	(debian)
		# Get:23 http://www-uxsup.csx.cam.ac.uk natty-updates/universe amd64 Packages [26.8 kB]
		# Get: 1 http://gb.archive.ubuntu.com intrepid/universe a2ps 1:4.14-1 [937kB]
		# Get: 1 http://gb.archive.ubuntu.com intrepid/main Translation-en_GB [27.0kB]
		# Get:4 http://www-uxsup.csx.cam.ac.uk natty-updates Release.gpg [198 B]
		# Get:3 http://www-uxsup.csx.cam.ac.uk natty/restricted Translation-en_GB [2,259 B]

		elsif (/^Get:\s*(\d+) (\S+) (\S+) (\S+) (\S+) \[(\S+)\]$/) { $got++; $seenG{$4} .= " G=$1:$2:$3:$5:$6"; }
		elsif (/^Get:\s*(\d+) (\S+) (\S+) (\S+) \[(\S+)\]$/) { $got++; $seenG{$4} .= " G=$1:$2:$3:$5"; }
		elsif (/^Get:\s*(\d+) http(\S+) (\S+) (\S+) \[([0-9.,]+\s*[kmKM]*B)\]$/) { $got++; $seenG{$4} .= " G=$1:$2:$3:$5"; }
		elsif (/^After this operation, \S+ of additional disk space will be used\.$/) { }
		elsif (/^\(Reading database ... \d+ files and directories currently installed\.\)$/) { }
		# Fetched 53.4MB in 1min9s (762kB/s)
		elsif (/^Fetched \S+ in \S+s \(\S+\)$/) { }
		elsif (/^Need to get \S+ of archives\.$/) { }
		elsif (/^Calculating upgrade\.\.\.$/) { }
		elsif (/^Installing new version of config file (\S+) \.\.\.$/) { $conf .= " $1"; }
		elsif (/^Setting up (\S+) \((\S+)\) \.\.\.$/) { $seen{$1} .= " SU=$2"; }
		elsif (/^Replacing files in old package (\S+) \.\.\.$/) { $seen{$1} .= " RF"; }
		elsif (/^Preparing to replace (\S+) (\S+) \(using (\S+)\) \.\.\.$/) { $seenU{$1} .= " PR=$2"; }
		elsif (/^Unpacking replacement (\S+) \.\.\.$/) { $seenU{$1} .= " Ur"; }
		elsif (/^Unpacking (\S+) \(from (\S+)\) \.\.\.$/) { $seenU{$1} .= " UP=$2"; }
		elsif (/^Selecting previously deselected package (\S+)\.$/) { $seen{$1} .= " Sel"; }
		elsif (/^Processing triggers for (\S+) \.\.\.$/) { $triggers .= " $1"; }
		#
		# Look for Dialog / Readline / Teletype fallback, and ignore "debconf" lines if other seen
		# -----------------------------------------------------------------------
		# debconf: Unable to initialise frontend: Dialog
		# debconf: unable to initialize frontend: Dialog
		# debconf: (TERM is not set, so the dialog frontend is not usable.)
		# debconf: falling back to frontend: Readline
		# debconf: unable to initialize frontend: Readline
		# debconf: (This frontend requires a controlling tty.)
		# debconf: falling back to frontend: Teletype
		# dpkg-preconfigure: unable to re-open stdin: 
		# Processing 1 changed doc-base file(s)...
		# Registering documents with scrollkeeper...
		# ldconfig deferred processing now taking place
		# -----------------------------------------------------------------------
		elsif (/^debconf: unable to initialize frontend: Dialog/) { $debconf .= $fl; }
		elsif (/^debconf: Unable to initialise frontend: Dialog/) { $debconf .= $fl; }
		elsif (/^debconf: Unable to initialize frontend: Dialog/) { $debconf .= $fl; }
		elsif (/^debconf: [uU]nable to initialize frontend: Dialog/) { $debconf .= $fl; }
		elsif (/^debconf: \(TERM is not set, so the dialog frontend is not usable.\)/) { $debconf .= $fl; }
		elsif (/^debconf: falling back to frontend: Readline/) { $debconf .= $fl; }
		elsif (/^debconf: unable to initialize frontend: Readline/) { $debconf .= $fl; }
		elsif (/^debconf: Unable to initialize frontend: Readline/) { $debconf .= $fl; }
		elsif (/^debconf: [Uu]nable to initialize frontend: Readline/) { $debconf .= $fl; }
		elsif (/^debconf: \(This frontend requires a controlling tty.\)/) { $debconf .= $fl; }
		elsif (/^debconf: falling back to frontend: Teletype/) { $debconf .= $fl; }
		elsif (/^debconf: \(Dialogue frontend will not work on a dumb terminal, an Emacs shell buffer, or without a controlling terminal.\)/) { $debconf .= $fl; }

		# Look for dpkg --configure problems, e.g.:
		# -----------------------------------------------------------------------
		# Configuration file `/etc/hp/hplip.conf'
		# ==> Modified (by you or by a script) since installation.
		# ==> Package distributor has shipped an updated version.
		#  What would you like to do about it ? Your options are:
		#  Y or I : install the package maintainer's version
		# N or O : keep your currently-installed version
		# D : show the differences between the versions
		# Z : background this process to examine the situation
		# The default action is to keep your current version.
		# *** hplip.conf (Y/I/N/O/D/Z) [default=N] ? dpkg: error processing hplip (--configure):
		# EOF on stdin at conffile prompt
		# No apport report written because the error message indicates its a followup error from a previous failure.
		# dpkg: dependency problems prevent configuration of hpijs:
		# hpijs depends on hplip (= 3.9.2-1ubuntu1); however:
		# Package hplip is not configured yet.
		# dpkg: error processing hpijs (--configure):
		# dependency problems - leaving unconfigured
		# -----------------------------------------------------------------------
		elsif (/==> Modified \(by you or by a script\) since installation./) { $conferrs .= $fl; s/^/**** /; $conferr++; }
		elsif (/==> Package distributor has shipped an updated version./) { $conferrs .= $fl; s/^/**** /; $conferr++; }
		elsif (/dpkg: error processing \S+ \(--configure\):/) { $conferrs .= $fl; s/^/**** /; $conferr++; }
		elsif (/EOF on stdin at conffile prompt/) { $conferrs .= $fl; s/^/**** /; $conferr++; }
		elsif (/Package \S+ is not configured yet./) { $conferrs .= $fl; s/^/**** /; $conferr++; }
		elsif (/dpkg: error processing \S+ \(--configure\):/) { $conferrs .= $fl; s/^/**** /; $conferr++; }
		elsif (/dependency problems - leaving unconfigured/) { $conferrs .= $fl; s/^/**** /; $conferr++; }
		# Start of collections
		elsif (/^The following NEW packages will be installed/) { $collecting = "NEW"; }
		elsif (/^Suggested packages:/) { $collecting = "Suggest"; }
		elsif (/^The following extra packages will be installed:/) { $collecting = "Extra"; }
		elsif (/^The following packages will be upgraded:/) { $collecting = "Upgrade"; }
		elsif (/^The following packages were automatically installed and are no longer required:/) { $collecting = "relics"; }
		elsif (/^The following packages have been kept back:/) { $collecting = "KeepBack"; }
		# End of collections
		elsif (/^ *( .*)/ && $collecting) { $val{$collecting} .= $1; }
		elsif (/^Use 'apt-get autoremove' to remove them\.$/ && $collecting eq "relics") { ; }
		else {
			$debconf = "" if /^dpkg-preconfigure: [uU]nable to re-open stdin:/;
			$collecting = "";
			$extras .= $fl;
		}
		#else { $bout .= $fl; }
	}
	unless (close(STDIN)) {
		$out = "# $cus: close failed on '$cmd'\n$out";
		$boring = -1;
		$exit++;
	}
	$rc = $?;

	my ($usedrepos) = "";
	foreach $repo (sort { $repos{$b} <=> $repos{$a} } keys %repos) {
		my ($name) = $repo;
		next unless $name;
		$name =~ s=http://==;
		$usedrepos .= " $repos{$repo} $name,";
	}
	$usedrepos =~ s/,$//;
	$repos .= "Repos:$usedrepos\n" if $usedrepos;

	$extras .= $debconf;
	$out = "Expected xfers: Got=$got\n$out" if $got && ! $todo;
	$out = "Expected xfers: Got=$got, TODO=$todo\n$out" if $todo;
	$full_log .= $out;

	$p = "";
	$p .= "$conferr Configure errors:\n$conferrs\n" if $conferrs;
	$p .= "Extras:\n$extras\n" if $extras;
	if ($op eq "upgrade") {
		$p .= "Conf:$conf\n" if $conf;
		# $p .= "Hit:$hit\n" if $hit;
		$p .= "Triggers:$triggers\n" if $triggers;
		$p .= "Mods:$mods\n" if $mods && $mods =~ /=[1-9]/;
		foreach $key (sort keys %val) {
			$p .= sprintf "%-20s%s\n", $key, $val{$key}
		}
		foreach $key (sort keys %seen) {
			$p .= sprintf "%-20s%s\n", $key, $seen{$key} if $key;
		}
	}
	$full_log .= $p . $repos;
	if ($p) { $p .= $repos; $repos = ""; }
	print wrap("", "", $p);

	# Look at the result to see if it is of interest
	unless ($boring) {
		$boring = -1 if $out =~ /No Match for argument/;
		$boring = -1 if $conferr;
	}

	# If not yet decided and no updates, decide it is boring
	unless ($boring) {
		$boring = 1 unless $todo || $got;
	}
	if ($out eq "") { $complete = ($trace) ? "" : " null"; }
	elsif ($out =~ s/\nComplete!\n/\n> Complete!\n/) {
	} else { ## What test??
		$complete = "";
		$complete .= " Got=$got" if $got;
		$complete .= " TODO=$todo" if $todo;
		$complete .= " Complete!";
		$out = $bout unless $verbose || $trace || $exit || $boring < 0;
	}
	if ($out) { $out .= $repos; $repos = ""; }
}

# Do the main "yum update"
sub do_yum_update {
	$quiet = ($trace) ? "" : " -e 0";
	$skip = " --skip-broken";
	$skip = '' if `rpm -q yum` =~ /yum-(\d+)\.(\d+)\.(\d+)/ && ($1<3||$2<2||$3<19);
	# Log changes if possible
	$log = (-e $changelog) ? " --changelog" : "";
	-e "/usr/sbin/yum-complete-transaction" &&
		&do_yum("${priv}yum-complete-transaction -y $quiet 2>&1", "complete");
	&do_yum("${priv}yum clean packages 2>&1", "clean");
	&do_yum("${priv}yum -y$quiet$log$skip update 2>&1", "update");
}

sub do_yum {
	my($cmd, $op) = @_;
	my($out) = "";
	my($bout) = "";
	my($boring) = 0;	# -1 is an error, 1 is "nothing to do", else 0
	my($partial) = 0;	# not all work done
	my($complete) = '';
	my($rc) = -99;

	# Run "yum update" and collect what we get
	&log($verbose, "# $cus: call: $cmd\n");
	$start = time;
	if ($trace) {
		$rc = system($cmd);
		warn("# $cus: $cmd' failed: $rc\n") if $rc;
	} else {
		open(STDIN, $cmd."|") || die("$cus: command '$cmd' failed: $!\n");
		while (<>) {
			s/\15$//;
			$out .= $_;
			chomp;
			if ($boutskip{$_}) { ; }
			elsif (/^\s*$/) {;}
			elsif (/^\s*Cleanup.*\d+\]\s*$/) {;}
			elsif (/^Loading "\S+" plugin$/) {;}
			elsif (/^---> Downloading header for \S+ to pack into transaction set.$/) {;}
			elsif (/^Excluding Packages from/) {;}
			elsif (/^Loaded plugins: /) {;}
	 		elsif (/^ \* \S+: \S+$/) {;}
	 		elsif (/^No Presto metadata available for \S+/) {;}
			else { $bout .= "$_\n"; }
		}
		unless (close(STDIN)) {
			$out = "# $cus: close failed on '$cmd'\n$out";
			$boring = -1;
			$exit++;
		}
		$rc = $?;

		# Look at the result to see if it is of interest
		unless ($boring) {
			$boring = -1 if $out =~ /No Match for argument/;
		}
		$partial = 1 if $out =~ /Skipped \(dependency problems\):/;
		$partial = 1 if $out =~ /Packages skipped because of dependency problems/;

		unless ($boring || $partial) {
			$boring = 1 if $out =~ /^0 packages removed/m;
			$boring = 1 if $out =~ /^0 package files removed/m;
			$boring = 1 if $out =~ /No unfinished transactions le/;
			$boring = 1 if $out =~ /Nothing to do/;
			$boring = 1 if $out =~ /No Packages marked for Update/;
		}
		$full_log .= $out;
	}
	my ($mostly) = ($partial) ? " Partially" : "";
	if ($out eq "") { $complete = ($trace) ? "" : " null"; }
	elsif ($out =~ s/\nComplete!\n/\n>$mostly Complete!\n/ ||
		$out =~ /Cleaning up completed transaction file/) {
		$complete = " Complete!";
		$out = $bout unless $verbose || $trace || $partial || $exit || $boring < 0;
	}

	# Decide whether to output anything
	my($dolog) = $verbose || $trace || $boring <= 0 || $partial;
	&log($dolog, "# $cus: '$cmd' ", time - $start, "s $rc $boring$mostly$complete:\n");
	(($dolog) ? $details : $basics) .= "-" x 30 . sprintf(" %-10s ", $op.$mostly) . "-" x 30 . "\n$out\n" if $out;
}

# -----------------------------------------------------------------------------

# Add any new RPMs

sub do_add {
	my($out) = "";
	my($boring) = 0;
	my($complete) = '';
	my($rc) = -99;
	my($cmd) = "${priv}cl-add-rpms -y -a 2>&1";

	 $cmd = "${priv}/etc/rc.scripts/fix.files 2>&1" if `cat /etc/issue` =~ /(Fedora Core release [123])|(Red Hat Linux release 9)/;

	# Run "yum install" and collect what we get
	&log($verbose, "# $cus: call: $cmd\n");
	$start = time;
	if ($trace) {
		$rc = system($cmd);
		warn("# $cus: $cmd' failed: $rc\n") if $rc;
	} else {
		open(STDIN, $cmd."|") || die("# $cus: command '$cmd' failed: $!\n");
		while (<>) { $out .= $_ }
		unless (close(STDIN)) {
			$out = "# $cus: close failed on '$cmd'\n$out";
			$boring = -1;
			$exit++;
		}
		$rc = $?;
		$full_log .= $out;

		unless ($boring) {
			$boring = 1 if $out eq "";
		}
	}
	if ($out eq "") { $complete = ($trace) ? "" : " null"; }
	elsif ($out =~ s/\nComplete!\n/\n> Complete!\n/) {
		$complete = " Complete!";
	}

	# Decide whether to output anything
	my($dolog) = $verbose || $trace || $boring <= 0;
	&log($dolog, "# $cus: '$cmd' ", time - $start, "s $rc $boring$complete:\n");
	(($dolog) ? $details : $basics) .= "-" x 30 . " add-rpms " . "-" x 30 . "\n$out\n" if $out;
}

# -----------------------------------------------------------------------------

# Run a simple command
sub run_cmd {
	my ($arg) = @_;
	my($out) = "";
	my($boring) = 0;
	my($cmd) = "${priv}$arg 2>&1";
	&log($verbose, "# $cus: call: $cmd\n");
	$start = time;
	if ($trace) {
		$rc = system($cmd);
		 warn("# $cus: $cmd' failed: $rc\n") if $rc;
	} else {
		open(STDIN, $cmd."|") || die("$cus: command '$cmd' failed: $!\n");
		while (<>) { $out .= $_ }
		unless (close(STDIN)) {
			$out = "# $cus: close failed on '$cmd'\n$out";
			$boring = -1;
			$exit++;
		}
		$full_log .= $out;
	}

	# Look at the result to see if it is of interest
	unless ($boring) {
		$boring = 1 if $out eq "";
	}

	# Decide whether to output anything
	my($dolog) = $verbose || $trace || $boring <= 0;
	&log($dolog, "# $cus: '$cmd' took ", time - $start, "s giving $boring:\n");
	(($dolog) ? $details : $basics) .= "-" x 30 . " $cmd " . "-" x 30 . "\n$out\n" if $out;
}

# -----------------------------------------------------------------------------

# Build an ownfiles

sub do_ownfiles {
	my($out) = "";
	my($interesting) = "";
	my($boring) = 0;
	my($cmd) = "${priv}cl-make-ownfiles -a 2>&1";

	# Run it and collect what we get
	&log($verbose, "# $cus: call: $cmd\n");
	$start = time;
	if ($trace) {
		$rc = system($cmd);
		 warn("# $cus: $cmd' failed: $rc\n") if $rc;
	} else {
		open(STDIN, $cmd."|") || die("$cus: command '$cmd' failed: $!\n");
		while (<>) { $out .= $_ }
		unless (close(STDIN)) {
			$out = "# $cus: close failed on '$cmd'\n$out";
			$boring = -1;
			$exit++;
		}
		$interesting = $out;
		$interesting =~ s/[^\n]*libGL error[^\n]*\n//g;
		$interesting =~ s/[^\n]*libGL warning:[^\n]*\n//g;
		$interesting =~ s/[^\n]*iptables[^\n]*\n//g;
		$interesting =~ s/[^\n]*Error: unable to open display[^\n]*\n//g;
		$interesting =~ s/[^\n]*No sensors found[^\n]*\n//g;
		$interesting =~ s/[^\n]*Can't access procfs[^\n]*\n//g;
		$interesting =~ s/[^\n]*Unable to find i2c bus information[^\n]*\n//g;
		$interesting =~ s/[^\n]* make sure you have mounted[^\n]*\n//g;
		$interesting =~ s/[^\n]*make sure you have done[^\n]*\n//g;
		$interesting =~ s/[^\n]*No sensors found[^\n]*\n//g;
		$interesting =~ s/[^\n]*Make sure you loaded all the kernel drivers you need[^\n]*\n//g;
		$interesting =~ s/[^\n]*Try sensors-detect to find out which these are.[^\n]*\n//g;
		chomp($interesting);

		# Look at the result to see if it is of interest
		unless ($boring) { $boring = 1 if $interesting eq ""; }
		$full_log .= $out;
	}

	# Decide whether to output anything
	my($dolog) = $verbose || $trace || $boring <= 0;
	&log($dolog, "# $cus: '$cmd' took ", time - $start, "s giving $boring:\n");
	(($dolog) ? $details : $basics) .= "-" x 30 . " ownfiles " . "-" x 30 . "\n$out\n" if$out;
	print "<<$interesting>>\n" if $interesting && ($verbose || $trace);
}

sub log {
	my ($print, @rest) = @_;
	print @rest if $print;
	$basics .= join("", @rest);
}
