#!/usr/bin/perl -w
# nagios: -epn

################################################
#
# Checks some TSM Stuff with dsmc (require report user)
#
# Thomas Sesselmann <t.sesselmann@dkfz.de> 2010
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
#
# Changelog:
# v0.9  2010.10.22 (ts)
#
####


use strict;
use warnings;
use Getopt::Long;   # libgetopt-mixed-perl perl-Getopt-Mixed

$ENV{LANG} = "C";


my %ERRORS=('OK'=>0,'WARNING'=>1,'CRITICAL'=>2,'UNKNOWN'=>3,'DEPENDENT'=>4);


our %opts = (
  help => 0,
  debug => 0,
  warning => undef,
  critical => undef,
  nodename => "",
  tsm_user => 'report',
  tsm_pass => 'xxx',
  tsm_server => '',
  tsm_errlog => '/tmp/tsmerr.log',
  timeout => 30,
);


my $dsmadmc="/usr/bin/dsmadmc";

my $check = "";
my %check_def=( 
  drives  => { 
    w => 0, c => 1,
    dsmq     => "SELECT count(*) AS num ,online,drive_state FROM drives GROUP BY online, drive_state",
    routine  => \&tsmcheck_drives,
    help_txt => "DRIVES\t\t - Number of Drives (with Online-State)",
    },
  tapes => {
    w => 0, c => 4,
    dsmq     => "SELECT count(*) AS num, access FROM volumes GROUP BY access",
    routine  => \&tsmcheck_tapes,
    help_txt => "TAPES\t\t - Number of Tapes (with Access-State)",
    },
  lib_tapes => {
    w => 20, c => 5,
    dsmq     => "SELECT count(*) AS num, status FROM libvolumes GROUP BY status",
    routine  => \&tsmcheck_libtapes,
    help_txt => "LIB_TAPES\t - Number of Tapes in the Libary (with State)",
    },
  pathes => {
    w => 0, c => 4,
    #dsmq     => "SELECT count(*) AS num, source_name, source_type, destination_type, library_name, online FROM paths WHERE destination_type = 'DRIVE' GROUP BY online, source_name, source_type, destination_type, library_name",
    dsmq     => "SELECT count(*) AS num, online FROM paths WHERE destination_type = 'DRIVE' GROUP BY online",
    routine  => \&tsmcheck_pathes,
    help_txt => "PATHES\t\t - Number of Pathes (with Access-State)",
    },
  admin_schedules => {
    w => 0, c => 4,
    dsmq     => "q event * t=a begind=today-1 begint=now endd=today endt=now",
    routine  => \&tsmcheck_admin_schedules,
    help_txt => "ADMIN_SCHEDULES\t - Number of (failed) Admin Schedules last 24hours",
  },
  client_schedules => {
    w => 200, c => 500,
    dsmq     => "q event * * t=c begind=today-1 begint=now endd=today endt=now",
    routine  => \&tsmcheck_client_schedules,
    help_txt => "CLIENT_SCHEDULES - Number of (failed) Client Schedules last 24hours",
  },
  moved_data => {
    dsmq     => "SELECT SUM(CAST(bytes/1024/1024 AS DECIMAL(8,2))) AS sum, activity FROM summary WHERE start_time>=current_timestamp - 1 hour GROUP BY activity",
    routine  => \&tsmcheck_data,
    help_txt => "MOVED_DATA\t - Calculate the moved data in the last hour.",
    opt      => "Moved Data last hour",
  },
  moved_data_24 => {
    dsmq     => "SELECT SUM(CAST(bytes/1024/1024 AS DECIMAL(8,2))) AS sum, activity FROM summary WHERE start_time>=current_timestamp - 1 day GROUP BY activity",
    routine  => \&tsmcheck_data,
    help_txt => "MOVED_DATA_24\t - Calculate the moved data in the last 24 hours.",
    opt      => "Moved Data last 24 hours",
  },
  all_data => {
    dsmq     => "select sum(num_files) AS FILES,sum(physical_mb) AS MB from occupancy",
    routine  => \&tsmcheck_all_data,
    help_txt => "ALL_DATA\t - Calculate all stored data.",
  },
  all_nodes => {
    dsmq     => "select sum(num_files) AS FILES,sum(physical_mb) AS MB, node_name from occupancy group by node_name",
    routine  => \&tsmcheck_print,
    help_txt => "ALL_NODES\t - Prints utilisation for all nodes (use with care!)",
  },
  nodeusage => {
    dsmq     => "SELECT SUM(num_files) AS FILES,SUM(physical_mb) AS MB, type FROM occupancy WHERE node_name='##NODENAME##' GROUP BY type",
    routine  => \&tsmcheck_hsm_data,
    help_txt => "NODEUSAGE\t - Checks the usage of the node '$opts{nodename}' (Backup and HSM)",
  },
  disk_pool => {
    w => 92, c => 95,
    dsmq     => "SELECT pct_utilized, pct_migr, stgpool_name FROM stgpools WHERE devclass='DISK'",
    routine  => \&tsmcheck_disk_pool,
    help_txt => "DISK_POOL\t - Check usage of the storagepools on disk",
  },
  tape_pool => {
    w => 1, c => 0,
    dsmq     => "SELECT pct_utilized, pct_migr, maxscratch, numscratchused, stgpool_name FROM stgpools WHERE devclass LIKE 'LTO%'",
    routine  => \&tsmcheck_tape_pool,
    help_txt => "TAPE_POOL\t - Check usage of the storagepools on tape (Scratch-Tapes)",
  },
  db_util => {
    w => 90, c => 95,
    dsmq     => "SELECT PCT_UTILIZED, MAX_PCT_UTILIZED, CAPACITY_MB FROM DB",
    routine  => \&tsmcheck_util,
    help_txt => "DB_UTIL\t\t - Checks the utilisation of the DB",
  },
  log_util => {
    w => 90, c => 95,
    dsmq     => "SELECT PCT_UTILIZED, MAX_PCT_UTILIZED, CAPACITY_MB FROM LOG",
    routine  => \&tsmcheck_util,
    help_txt => "LOG_UTIL\t - Checks the utilisation of the LOG",
  },
  test => { 
    #dsmq     => "SELECT count(*) AS num, online, destination_type FROM paths GROUP BY online,destination_type",
    dsmq     => "select SUBSTR(activity,1,15) as activity, avg( cast(bytes as decimal(18,0)) / cast((end_time-start_time) seconds as decimal(18,0))) as b_per_sec from summary where bytes >= 1000 and start_time>=current_timestamp-30 days group by activity",
    routine  => \&tsmcheck_print,
    },


  );

# HELP:
#dsmadmc -se=ext_backup -id=report -password=adsm -displaymode=table
# select * from syscat.tables
# select * from drives



sub start_stuff();
sub print_exit($$);
sub check_tsm($);


## get params
$check = start_stuff();

## Check
check_tsm($check);




#####################################################


### Start stuff
sub start_stuff()
{

  ## get options
  Getopt::Long::Configure("no_ignore_case");
  Getopt::Long::GetOptions( \%opts, 
    'help|h',
    'debug|d',
    'warning|w=s',
    'critical|crit|c=s',
    'nodename=s',
    'tsm_user|tsm-user=s',
    'tsm_pass|tsm-pass=s',
    'tsm_server|tsm-server=s',
    'tsm_errlog|tsm-errlog=s',
    'timeout|t=i',
    ) or $opts{help}=1;


  ## checks:
  $opts{help} = 1 if ( scalar @ARGV != 1 );
  $opts{nodename} = uc $opts{nodename};
  my $check = lc shift @ARGV;
  if ( exists $check_def{$check} ) {
    $check = $check_def{$check};
  } else {
    $opts{help}=1;
  }


  ## Usage:
  if ( $opts{help} )  {
    my $usage = "$0 [Options]* [Check]
Options:
     -h, --help              print this help message
     -w, --warning=<int>     Warning Level for some checks
     -c, --critical=<int>    Critical Level for some checks
     -d, --debug             prints debug output (use only to test plugin, Default: '$opts{debug}')
     --nodename=<string>     Nodename (used for some queries (Default: '$opts{nodename}')
     --tsm-user=<string>     TSM-User with ANALYST rights (Default: '$opts{tsm_user}')
     --tsm-pass=<string>     TSM-Password for TSM-User (Default: '$opts{tsm_pass}')
     --tsm-server=<string>   TSM-Server (defined in dsm.sys) (Default: '$opts{tsm_server}')
     --tsm-errlog=<string>   TSM-Errlog writeable by this user (Default: '$opts{tsm_errlog}')
     -t, --timeout=<d>   Timeout for this script (Default: '$opts{timeout}')
Check:";
    foreach my $c ( sort keys %check_def ) {
       if ( exists $check_def{$c}{help_txt} ) {
         $usage .= "\t$check_def{$c}{help_txt}\n";
       }
    }

    die "$usage\n";
  }


  return $check;
}


### Print Message and Exit with result-code
sub print_exit($$) 
{
  my $msg = shift;
  my $return_code = shift;

  print "$msg";
  exit $return_code;
}


### Checks and run the evaluation routine ... 
sub check_tsm($)
{
  my $check = shift;

  $opts{tsm_server} = "-server=$opts{tsm_server}" if "$opts{tsm_server}" ne "";
  $opts{tsm_errlog} = "-errorlogname=$opts{tsm_errlog}" if "$opts{tsm_errlog}" ne "";

  $check->{dsmq} =~ s/##NODENAME##/$opts{nodename}/g;

  my $cmd = "$dsmadmc $opts{tsm_server} -id=$opts{tsm_user} -password=$opts{tsm_pass} -displaymode=table $opts{tsm_errlog} "
           .'"'.$check->{'dsmq'}.'"';


  ## timeout
  local $SIG{'ALRM'} = sub { 
    #kill $childpid;
    #sleep 2;
    #kill 9,$childpid;
    print_exit("CRITICAL - timeout after $opts{timeout} seconds.\n",$ERRORS{CRITICAL});
  };
  alarm($opts{timeout});


  ## start process/cmd
  my @tsm_out = ();
  my @tsm_err = ();
  my $afterhead = 0;
  open TMP, "$cmd |" or die "Error: Can't open pipe for command '$cmd': $!\n";
  while(<TMP>) { # ignore header stuff
    chomp $_;
    print "tsm_output>$_<\n" if $opts{debug};
    if ( $_ =~ /^ANS8000I/ ){ # Server command: ...
      $afterhead = 1;
      next;
    }
    next if ( $_ =~ /^\s*$/ );
    if ( $_ =~ /^ANS\d{4}E/ ) {
      push @tsm_err, $_;
    }
    last if ( $_ =~ /^ANS8002I/ ); # Highest return code was ...
    push @tsm_out, $_ if $afterhead;
  }
  close TMP;


  ## timeout off
  alarm(0);	


  ## calc result ..
  if ( defined $opts{warning} ) { 
    $check->{w} = $opts{warning};
  } elsif ( not exists $check->{w} ) {
    $check->{w} = 1;
  }
  if ( defined $opts{critical} ) { 
    $check->{c} = $opts{critical}; 
  } elsif ( not exists $check->{c} ) {
    $check->{c} = 0;
  }

  my $status = "OK";
  my $out = "";
  my $perf_data = "";
  if ( @tsm_err ) { # ERRORS
    $status = "WARNING";
    if ( grep /^ANS8023E/, @tsm_err ) { #Unable to establish session with server
      $status = "UNKNOWN";
      $out .= "No Connection to Server! ";
    }
    if ( grep /^ANS1316E/, @tsm_err ) { #The server does not have enough recovery log space to continue the current operation
      $status = "CRITICAL";
      $out .= "LOG DB is Full! ";
    }
    $out .= "(Errors: ".join(",",map { s/^(ANS\d{4}E).*/$1/g; $_ } @tsm_err ).")";

  } else { # OK
    if ( exists $check->{routine} ) {
      ( $status, $out, $perf_data ) = &{ $check->{routine} }(\@tsm_out,$check->{w},$check->{c},$check->{opt});
    } else {
      #TODO ?
      $status = "UNKNOWN";
      $out = "No routine for evaluation defined!";
    }
  }
  print_exit("$status - $out|$perf_data\n",$ERRORS{$status});

}


### Parse Template Line with [- ] from TSM Output
sub parsetemplateline($)
{
  my $line = shift; # line with '-' and ' '

  return () if not defined $line;
  return () if ( $line !~ /^[- ]+$/ );

  my @map = ();
  my $ac = "-";
  my $an = 0;
  for (my $i = 0; $i < length $line; $i++) {
    my $c = substr $line, $i, 1;
    if ( $ac eq $c ) {
      $an++;
    } else {
      push @map, { char=>$ac, num=>$an };
      $ac = $c;
      $an = 1;
    }
  }
  push @map, { char=>$ac, num=>$an };

  return @map;
}


## Dummy routine to test or debug or print 
sub tsmcheck_print($$$)
{
  my @tsm = @{ ( shift ) };
  my $w = shift;
  my $c = shift; 

  my $status = "OK";
  my $out = "Dummy Routine";
  my $perf_data = "";

  foreach ( @tsm ) {
    if ( $opts{debug} ) {
      print ">$_<\n";
    } else {
      print "$_\n";
    }
  }

  return $status, $out, $perf_data;
}


## Check Drives (and State)
sub tsmcheck_drives($$$)
{
  my @tsm = @{ ( shift ) };
  my $w = shift;
  my $c = shift; 

  my $status = "OK";
  my $out = "";
  my $perf_data = "";

  shift @tsm; # header
  shift @tsm; # ---- ---
  my $online  = 0;
  my $offline = 0;
  my %state = ( empty=>0, unavailable=>0, loaded=>0, unloaded=>0, reserved=>0, unknown=>0 );
  # Unavailable
  #    The drive is not available to the library for operations.
  # Empty
  #    The drive is empty and ready for operations.
  # Loaded
  #    The drive is currently loaded, and the server is performing
  #    operations to the drive.
  # Unloaded
  #    The media has been ejected from the drive.
  # Reserved
  #    The drive is reserved for a mount request.
  # Unknown
  #    The drive begins in drive state unknown as a result of being
  #    defined, as a result of server initialization, or as a result of
  #    having its status updated to online.
  foreach ( @tsm ) {
    if ( $_ =~ /^\s*(\d+)\s+(\w+)\s+(\w+)/ ) {
      if ( $2 eq "YES" ) { 
        $online += $1; 
      } else {
        $offline += $1;
      }
      if ( exists $state{lc($3)} ) {
        $state{lc($3)} += $1;
      }
    }
  }
  my $sum_drives = $online + $offline;

  if ( $state{empty} == 0 ) { $status = "WARNING"; }
  if ( $offline > $w ) { $status = "WARNING"; }
  if ( $offline > $c ) { $status = "CRITICAL"; }

  $out .= sprintf "%d/%d Drives are Online (%d drives empty)",$online,$sum_drives,$state{empty};
  # akt; w; c; min; max;
  $perf_data .= sprintf "drives=%d;%d;%d;%d;%d ",$online,$sum_drives-$w,$sum_drives-$c,0,$sum_drives;
  foreach ( qw/empty unavailable loaded unloaded reserved unknown/ ) {
    $perf_data .= sprintf "%s=%d ",$_,$state{$_};
  }

  return $status, $out, $perf_data;
}


## Check Tapes (and State)
sub tsmcheck_tapes($$$)
{
  my @tsm = @{ ( shift ) };
  my $w = shift;
  my $c = shift; 

  my $status = "OK";
  my $out = "";
  my $perf_data = "";

  shift @tsm; # header
  shift @tsm; # ---- ---
  my %tapes = ( READWRITE=>0, OFFSITE=>0, UNAVAILABLE=>0, READONLY=>0, DESTROYED=>0 );
  my $sum = 0;
  my $errors = 0;
  foreach ( @tsm ) {
    if ( $_ =~ /^\s*(\d+)\s+(\w+)/ ) {
      $tapes{$2} = $1;
      $sum += $1;
    }
  }

  $out .= sprintf "%d Tapes (",$sum;
  $perf_data .= sprintf "ALL=%d ",$sum;
  foreach my $k ( sort keys %tapes ) {
    if ( $tapes{$k} > 0 ) { $out .= sprintf "%d %s, ",$tapes{$k},$k; }
    $perf_data .= sprintf "%s=%d ",$k,$tapes{$k};
    if ( ($k ne "READWRITE") and ($k ne "OFFSITE") ){ $errors += $tapes{$k}; }
  }
  $out =~ s/, $/)/;
  $out =~ s/ \($//;  # if no tapes


  if ( $errors > $w ) { $status = "WARNING"; }
  if ( $errors > $c ) { $status = "CRITICAL"; }

  return $status, $out, $perf_data;
}


## Check Pathes (and State)
sub tsmcheck_pathes($$$)
{
  my @tsm = @{ ( shift ) };
  my $w = shift;
  my $c = shift; 

  my $status = "OK";
  my $out = "";
  my $perf_data = "";

  shift @tsm; # header
  shift @tsm; # ---- ---
  my $online  = 0;
  my $offline = 0;
  foreach ( @tsm ) {
    if ( $_ =~ /^\s*(\d+)\s+(\w+)/ ) {
      if ( $2 eq "YES" ) { 
        $online += $1; 
      } else {
        $offline += $1;
      }
    }
  }
  my $sum_drives = $online + $offline;

  if ( $offline > $w ) { $status = "WARNING"; }
  if ( $offline > $c ) { $status = "CRITICAL"; }

  $out .= sprintf "%d/%d Pathes are Online",$online,$sum_drives;
  # akt; w; c; min; max;
  $perf_data .= sprintf "pathes=%d;%d;%d;%d;%d ",$online,$sum_drives-$w,$sum_drives-$c,0,$sum_drives;

  return $status, $out, $perf_data;
}


## Check Tapes from Libary (and Number of Scratch Tapes)
sub tsmcheck_libtapes($$$)
{
  my @tsm = @{ ( shift ) };
  my $w = shift;
  my $c = shift; 

  my $status = "OK";
  my $out = "";
  my $perf_data = "";

  shift @tsm; # header
  shift @tsm; # ---- ---
  my %tapes = ( Private=>0, Scratch=>0, Cleaner=>0 );
  my $sum  = 0;
  foreach ( @tsm ) {
    if ( $_ =~ /^\s*(\d+)\s+(\w+)/ ) {
      $tapes{$2} = $1;
      $sum += $1;
    }
  }

  $out .= sprintf "%d Tapes in Libary (",$sum;
  $perf_data .= sprintf "ALL=%d ",$sum;
  foreach my $k ( sort keys %tapes ) {
    if ( $tapes{$k} > 0 or $k eq "Scratch" ) { $out .= sprintf "%d %s, ",$tapes{$k},$k; }
    $perf_data .= sprintf "%s=%d ",$k,$tapes{$k};
  }
  $out =~ s/, $/)/;
  $out =~ s/ \($//;  # if no tapes


  if ( $tapes{'Scratch'} < $w ) { $status = "WARNING"; }
  if ( $tapes{'Scratch'} < $c ) { $status = "CRITICAL"; }

  return $status, $out, $perf_data;
}


## Check Admin Schedules for Failed
sub tsmcheck_admin_schedules($$$)
{
  my @tsm = @{ ( shift ) };
  my $w = shift;
  my $c = shift; 

  my $status = "OK";
  my $out = "";
  my $perf_data = "";

  shift @tsm; # header
  my @map = parsetemplateline( shift @tsm ); # ---- ---
  #my %failed = ();
  my %nums = ( Completed=>0, Failed=>0, Missed=>0, Started=>0 );
  my $sum = 0;
  my @oldline = ('','','','');
  foreach ( @tsm ) {
    if ( $_ =~ /^(.{$map[0]{num}})\s{$map[1]{num}}(.{$map[2]{num}})\s{$map[3]{num}}(.{$map[4]{num}})\s{$map[5]{num}}(.{$map[6]{num}})/ ) {
      my @newline = ($1,$2,$3,$4);
      if ( $newline[0] =~ /^\s*$/ ) { # break line
        if ( $newline[2] =~ /^\s\w+/ ) { 
          $oldline[2] =~ s/-$//;
          $newline[2] =~ s/^\s//;
          $oldline[2] .= $newline[2];
        }
      } else {
        if ( $oldline[0] ne '' ) {
          $oldline[3] =~ s/^(\w+)\s.*$/$1/;
          #$oldline[2] =~ s/\s*$//;
          $nums{$oldline[3]}++;
          $sum++;
          #if ( "$oldline[3]" eq "Failed" ) { $failed{$oldline[2]} = 1; }
        }
        @oldline = @newline;
      }
    }
  }
  if ( $oldline[0] ne '' ) {
    $oldline[3] =~ s/^(\w+)\s.*$/$1/;
    #$oldline[2] =~ s/\s*$//;
    $nums{$oldline[3]}++;
    $sum++;
    #if ( "$oldline[3]" eq "Failed" ) { $failed{$oldline[2]} = 1; }
  }

  $out .= sprintf "%d Schedules Run (",$sum;
  $perf_data .= sprintf "ALL=%d ",$sum;
  foreach my $k ( sort keys %nums ) {
    if ( $nums{$k} > 0 ) { $out .= sprintf "%d %s, ",$nums{$k},$k; }
    $perf_data .= sprintf "%s=%d ",$k,$nums{$k};
  }
  $out =~ s/, $/)/;
  $out =~ s/ \($//;  # if no tapes


  if ( $nums{'Failed'}+$nums{'Missed'} > $w ) { $status = "WARNING"; }
  if ( $nums{'Failed'}+$nums{'Missed'} > $c ) { $status = "CRITICAL"; }

  return $status, $out, $perf_data;
}


## Check Client Schedules for Failed/Missed
sub tsmcheck_client_schedules($$$)
{
  my @tsm = @{ ( shift ) };
  my $w = shift;
  my $c = shift; 

  my $status = "OK";
  my $out = "";
  my $perf_data = "";

  shift @tsm; # header
  my @map = parsetemplateline( shift @tsm ); # ---- ---
  #my %failed = ();
  my %nums = ( Completed=>0, Failed=>0, Missed=>0, Started=>0, Pending=>0, InProgress=>0, Severed=>0 );
  my $sum = 0;
  my @oldline = ('','','','','');
  foreach ( @tsm ) {
    if ( $_ =~ /^(.{$map[0]{num}})\s{$map[1]{num}}(.{$map[2]{num}})\s{$map[3]{num}}(.{$map[4]{num}})\s{$map[5]{num}}(.{$map[6]{num}})\s{$map[7]{num}}(.{$map[8]{num}})/ ) {
      my @newline = ($1,$2,$3,$4,$5);
      #print "debug>$1,$2,$3,$4,$5<\n";
      if ( $newline[0] =~ /^\s*$/ ) { # break line
        if ( $newline[3] =~ /^\s\w+/ ) { 
          $oldline[3] =~ s/-$//;
          $newline[3] =~ s/^\s//;
          $oldline[3] .= $newline[3];
        }
      } else {
        if ( $oldline[0] ne '' ) {
          if ( $oldline[4] =~ /In Progr-/ ) { $oldline[4] = "InProgress"; }
          $oldline[4] =~ s/^(\w+)\s.*$/$1/;
          #$oldline[3] =~ s/\s*$//;
          $nums{$oldline[4]}++;
          $sum++;
          #if ( "$oldline[4]" eq "Failed" ) { $failed{$oldline[3]} = 1; }
        }
        @oldline = @newline;
      }
    }
  }
  if ( $oldline[0] ne '' ) {
    $oldline[4] =~ s/^(\w+)\s.*$/$1/;
    #$oldline[3] =~ s/\s*$//;
    $nums{$oldline[4]}++;
    $sum++;
    #if ( "$oldline[4]" eq "Failed" ) { $failed{$oldline[3]} = 1; }
  }

  $out .= sprintf "%d Schedules Run (",$sum;
  $perf_data .= sprintf "ALL=%d ",$sum;
  foreach my $k ( sort keys %nums ) {
    if ( $nums{$k} > 0 ) { $out .= sprintf "%d %s, ",$nums{$k},$k; }
    $perf_data .= sprintf "%s=%d ",$k,$nums{$k};
  }
  $out =~ s/, $/)/;
  $out =~ s/ \($//;  # if no tapes


  if ( $nums{'Failed'}+$nums{'Missed'} > $w ) { $status = "WARNING"; }
  if ( $nums{'Failed'}+$nums{'Missed'} > $c ) { $status = "CRITICAL"; }

  return $status, $out, $perf_data;
}


## Check Moved Data (last hours)
sub tsmcheck_data($$$;$)
{
  my @tsm = @{ ( shift ) };
  my $w = shift;
  my $c = shift;
  my $otext = shift; $otext = "Moved Data last hour" if not defined $otext;

  my $status = "OK";
  my $out = "";
  my $perf_data = "";

  shift @tsm; # header
  my @map = parsetemplateline( shift @tsm ); # ---- ---
  my %data = ( BACKUP=>0, ARCHIVE=>0, EXPIRATION=>0, FULL_DBBACKUP=>0, INCR_DBBACKUP=>0, MIGRATION=>0, 
               RECLAMATION=>0, RESTORE=>0, RETRIEVE=>0, NAS_Backup_to_TSM_Storage=>0, STGPOOL_BACKUP=>0 );
  my @oldline = ('','');
  foreach ( @tsm ) {
    if ( $_ =~ /^(.{$map[0]{num}})\s{$map[1]{num}}(.{$map[2]{num}})/ ) {
      my @newline = ($1,$2);
      #print "debug>$1,$2<\n";
      if ( $newline[0] =~ /^\s*$/ ) { # break line
        if ( $newline[1] =~ /^\s\w+/ ) { 
          $oldline[1] =~ s/-$//;
          $newline[1] =~ s/^\s//;
          $oldline[1] .= $newline[1];
        }
      } else {
        #print "debug2>$oldline[0],$oldline[1]\n";
        if ( $oldline[0] ne '' ) {
          $oldline[1] =~ s/\s*$//;
          $oldline[1] =~ s/\s+/_/g;
          $data{$oldline[1]} += $oldline[0];
        }
        @oldline = @newline;
      }
    }
  }
  if ( $oldline[0] ne '' ) {
    $oldline[1] =~ s/\s*$//;
    $oldline[1] =~ s/\s+/_/g;
    $data{$oldline[1]} += $oldline[0];
  }

  $out .= "$otext: ";
  foreach my $k ( sort keys %data ) {
    if ( ( $k =~ /^(ARCHIVE|BACKUP|MIGRATION|NAS_Backup)/ ) and ( $data{$k} > 10 ) ){ 
      $out .= sprintf "%3.1fGB %s, ",$data{$k}/1024,$k;
    }
    $perf_data .= sprintf "%s=%dMB ",$k,$data{$k};
  }
  $out =~ s/, $//;


  return $status, $out, $perf_data;
}


## Check all stored data
sub tsmcheck_all_data($$$)
{
  my @tsm = @{ ( shift ) };
  my $w = shift;
  my $c = shift; 

  my $status = "OK";
  my $out = "";
  my $perf_data = "";

  shift @tsm; # header
  shift @tsm; # ---- ---
  my $files = 0;
  my $data  = 0;
  foreach ( @tsm ) {
    if ( $_ =~ /^\s*(\d+)\s+([\d\.]+)/ ) {
      $files = $1;
      $data = $2;
    }
  }
  my $unit = "MB";
  $perf_data .= sprintf "data=%dMB ",$data;
  $perf_data .= sprintf "files=%d ",$files;
  if ( $data > 10000 ) { $data /= 1024; $unit = "GB"; }
  if ( $data > 10000 ) { $data /= 1024; $unit = "TB"; }
  $files =~ s/(\d)(\d{3})(\,|$)/$1,$2$3/g;
  $files =~ s/(\d)(\d{3})(\,|$)/$1,$2$3/g;
  $files =~ s/(\d)(\d{3})(\,|$)/$1,$2$3/g;
  $files =~ s/(\d)(\d{3})(\,|$)/$1,$2$3/g;
  $out .= sprintf "All stored Data: %6.2f %s, %s files",$data,$unit,$files;

  return $status, $out, $perf_data;
}


## Check all stored data
sub tsmcheck_hsm_data($$$)
{
  my @tsm = @{ ( shift ) };
  my $w = shift;
  my $c = shift; 

  my $status = "OK";
  my $out = "";
  my $perf_data = "";

  shift @tsm; # header
  shift @tsm; # ---- ---
  my $b_files = 0;
  my $b_data  = 0;
  my $s_files = 0;
  my $s_data  = 0;
  foreach ( @tsm ) {
    if ( $_ =~ /^\s*(\d+)\s+([\d\.]+)\s+Bkup/ ) {
      $b_files = $1;
      $b_data = $2;
    }
    if ( $_ =~ /^\s*(\d+)\s+([\d\.]+)\s+SpMg/ ) {
      $s_files = $1;
      $s_data = $2;
    }
  }
  my $b_unit = "MB";
  my $s_unit = "MB";
  $perf_data .= sprintf "backup_data=%dMB ",$b_data;
  $perf_data .= sprintf "backup_files=%d ",$b_files;
  $perf_data .= sprintf "hsm_data=%dMB ",$s_data;
  $perf_data .= sprintf "hsm_files=%d ",$s_files;
  if ( $b_data > 10000 ) { $b_data /= 1024; $b_unit = "GB"; }
  if ( $b_data > 10000 ) { $b_data /= 1024; $b_unit = "TB"; }
  if ( $s_data > 10000 ) { $s_data /= 1024; $s_unit = "GB"; }
  if ( $s_data > 10000 ) { $s_data /= 1024; $s_unit = "TB"; }
  $b_files =~ s/(\d)(\d{3})(\,|$)/$1,$2$3/g;
  $b_files =~ s/(\d)(\d{3})(\,|$)/$1,$2$3/g;
  $b_files =~ s/(\d)(\d{3})(\,|$)/$1,$2$3/g;
  $b_files =~ s/(\d)(\d{3})(\,|$)/$1,$2$3/g;
  $s_files =~ s/(\d)(\d{3})(\,|$)/$1,$2$3/g;
  $s_files =~ s/(\d)(\d{3})(\,|$)/$1,$2$3/g;
  $s_files =~ s/(\d)(\d{3})(\,|$)/$1,$2$3/g;
  $s_files =~ s/(\d)(\d{3})(\,|$)/$1,$2$3/g;
  $out .= sprintf "Backup Data: %6.2f %s (%s files), ",$b_data,$b_unit,$b_files;
  $out .= sprintf "HSM Data: %6.2f %s (%s files) ",$s_data,$s_unit,$s_files;

  return $status, $out, $perf_data;
}


## Check usage of the disk storage pools
sub tsmcheck_disk_pool($$$)
{
  my @tsm = @{ ( shift ) };
  my $w = shift;
  my $c = shift; 

  my $status = "OK";
  my $out = "";
  my $perf_data = "";

  shift @tsm; # header
  my @map = parsetemplateline( shift @tsm ); # ---- ---
  my %data = ();
  my @oldline = ('','','');
  foreach ( @tsm ) {
    if ( $_ =~ /^(.{$map[0]{num}})\s{$map[1]{num}}(.{$map[2]{num}})\s{$map[3]{num}}(.{$map[4]{num}})/ ) {
      my @newline = ($1,$2,$3);
      #print "debug>$1,$2,$3<\n";
      if ( $newline[0] =~ /^\s*$/ ) { # break line
        if ( $newline[2] =~ /^\s\w+/ ) { 
          $oldline[2] =~ s/-$//;
          $newline[2] =~ s/^\s//;
          $oldline[2] .= $newline[2];
        }
      } else {
        #print "debug2>$oldline[0],$oldline[1],$oldline[2]\n";
        if ( $oldline[0] ne '' ) {
          $oldline[2] =~ s/\s*$//;
          $oldline[2] =~ s/\s+/_/g;
          $data{$oldline[2]} = $oldline[0];
        }
        @oldline = @newline;
      }
    print "tsm_output>$_<\n" if $opts{debug};
    }
  }
  if ( $oldline[0] ne '' ) {
    $oldline[2] =~ s/\s*$//;
    $oldline[2] =~ s/\s+/_/g;
    $data{$oldline[2]} = $oldline[0];
  }

  $out .= "Usage of STGpools (disk-only >1%): ";
  foreach my $k ( sort keys %data ) {
    if ( $data{$k} > 1 ){ 
      $out .= sprintf "%3.1f%% %s, ",$data{$k},$k;
    }
    $perf_data .= sprintf "%s=%d ",$k,$data{$k};
    if (( $data{$k} > $w ) and ( $ERRORS{$status} < $ERRORS{WARNING} ) )  { $status = "WARNING"; }
    if (( $data{$k} > $c ) and ( $ERRORS{$status} < $ERRORS{CRITICAL} ) ) { $status = "CRITICAL"; }
  }
  $out =~ s/, $//;

  return $status, $out, $perf_data;
}


## Check usage of the disk storage pools
sub tsmcheck_tape_pool($$$)
{
  my @tsm = @{ ( shift ) };
  my $w = shift;
  my $c = shift; 

  my $status = "OK";
  my $out = "";
  my $perf_data = "";

  shift @tsm; # header
  my @map = parsetemplateline( shift @tsm ); # ---- ---
  my %data = ();
  my @oldline = ('','','','','');
  foreach ( @tsm ) {
    if ( $_ =~ /^(.{$map[0]{num}})\s{$map[1]{num}}(.{$map[2]{num}})\s{$map[3]{num}}(.{$map[4]{num}})\s{$map[5]{num}}(.{$map[6]{num}})\s{$map[7]{num}}(.{$map[8]{num}})/ ) {
      my @newline = ($1,$2,$3,$4,$5);
      #print "debug>$1,$2,$3,$4,$5<\n";
      if ( $newline[0] =~ /^\s*$/ ) { # break line
        if ( $newline[4] =~ /^\s\w+/ ) { 
          $oldline[4] =~ s/-$//;
          $newline[4] =~ s/^\s//;
          $oldline[4] .= $newline[4];
        }
      } else {
        #print "debug2>$oldline[0],$oldline[1],$oldline[2],$oldline[3],$oldline[4]\n";
        if ( $oldline[0] ne '' ) {
          $oldline[4] =~ s/\s*$//;
          $oldline[4] =~ s/\s+/_/g;
          $data{$oldline[4]} = { util=>$oldline[0], migr=>$oldline[1], maxscratch=>$oldline[2], usedscratch=>$oldline[3] };
        }
        @oldline = @newline;
      }
    }
  }
  if ( $oldline[0] ne '' ) {
    $oldline[4] =~ s/\s*$//;
    $oldline[4] =~ s/\s+/_/g;
    $data{$oldline[4]} = { util=>$oldline[0], migr=>$oldline[1], maxscratch=>$oldline[2], usedscratch=>$oldline[3] };
  }

  $out .= "Usage of STGpools (tape-only >10Tapes): ";
  foreach my $k ( sort keys %data ) {
    if ( $data{$k}{maxscratch} > 10 ){ 
      $out .= sprintf "%d/%d %s (util: %3.1f%%), ",$data{$k}{usedscratch}, $data{$k}{maxscratch}, $k, $data{$k}{util};
      if (( $data{$k}{maxscratch}-$data{$k}{usedscratch} <= $w ) and ( $ERRORS{$status} < $ERRORS{WARNING} ) )  { $status = "WARNING"; }
      if (( $data{$k}{maxscratch}-$data{$k}{usedscratch} <= $c ) and ( $ERRORS{$status} < $ERRORS{CRITICAL} ) ) { $status = "CRITICAL"; }
    }
    $perf_data .= sprintf "%s\_util=%d %s\_tapes=%d;%d;%d;%d;%d ", $k, $data{$k}{util}, $k, $data{$k}{usedscratch}, 
                           $data{$k}{maxscratch}-$w, $data{$k}{maxscratch}-$c, 0, $data{$k}{maxscratch};
  }
  $out =~ s/, $//;

  return $status, $out, $perf_data;
}


## Check utilisation of DB and/or LOG
sub tsmcheck_util($$$)
{
  my @tsm = @{ ( shift ) };
  my $w = shift;
  my $c = shift; 

  my $status = "OK";
  my $out = "";
  my $perf_data = "";

  shift @tsm; # header
  shift @tsm; # ---- ---
  my $util = 0;
  my $max  = 0;
  my $cap  = 0;
  foreach ( @tsm ) {
    if ( $_ =~ /^\s*([\d.]+)\s+([\d.]+)\s+([\d.]+)/ ) {
      $util = $1;
      $max  = $2;
      $cap  = $3;
    }
  }

  $out .= sprintf "%4.1f%% Utilisation (max: %4.1f%%, Capacity: %dMB)",$util,$max,$cap;
  $perf_data .= sprintf "util=%.2f;%d;%d max=%.2f cap=%dMB ",$util,$w,$c,$max,$cap;
  
  if ( $util > $w ) { $status = "WARNING"; }
  if ( $util > $c ) { $status = "CRITICAL"; }

  return $status, $out, $perf_data;
}