#!/opt/perl/bin/perl
use strict;
$ENV{ARCH}='Linux86';
use GPS::DATES;
use GPS::Defaults;
use Cwd;
#use Smart::Comments;
use Getopt::Long;
use Set::Scalar;
use Net::FTP;
use Net::FTP::File;
use LWP::Simple;
use File::Basename;
use File::Copy;
use File::Path;
use Carp;
my $progname = basename $0;
my $usage = <<EOT;
  $progname proj date [end_date]
EOT
my($proj,$sd,$ed,$archive,$help);
my $args = @ARGV;
GetOptions ( "proj:s"  => \$proj,
	     "sdate:s" => \$sd,
	     "edate:s" => \$ed,
	     "archive" => \$archive,
	     "help"    => \$help
	   ) or die "$progname : Error in options";

if ( ! $args || $help ) {
	die $usage;
}

my %par = get_defaults( );
if (! $ed ) {
	$ed = $sd;
}

#wait for pbo process to finish
my $force = 0;
my @ps = grep { ! /$$/ } map { (split($_))[1] } grep { /get_.*rnx_data/ } `ps -cfe`;
print "ps:|@ps|\n";

@ps = () if $force;

while ( @ps ) {
  sleep (6+rand(4));
  #@ps = grep { /get_.*rnx_data/ } `ps -cfe`;
  @ps = grep { ! /$$/ } map { (split($_))[1] } grep { /get_.*rnx_data/ } `ps -cfe`;
  print "ps:|@ps|\n";
}
if ( $proj eq 'panga' ) {
   #`mk_panga_nets ''`;
   my $pnsf = "$ENV{GPS_HOME_DIR}/etc/proj_lists/panga_net_sites";
   copy ("$pnsf.full", $pnsf);
}
my @s = `get_sites_from_proj_list $proj`; chomp @s;
my $sites = Set::Scalar->new();
$sites->insert(map { lc } @s);
my $today = get_date('today','cal,num');
my $jd = get_date( $sd,'jd');
my $ejd = get_date($ed,'jd');
die "bad dates $sd > $ed \n" if $ejd < $jd;

while ( $jd <= $ejd) {
   get_rinex($jd,$sites);
   $jd++;
}

sub archived {
   my $yyyy=shift;
   my $doy =shift;
   my $dir;
   if ($archive) {
       $dir = "$par{RNX_DIR_ARCHIVE}/$yyyy/$doy";
   } else {
       $dir = "$par{RNX_DIR}/$yyyy/$doy";
   }
   my $yy = unpack 'x2a2',$yyyy;
   my $sites = Set::Scalar->new();
   if ( -d $dir ) {
     opendir(my $dh,$dir) or die "Couldn't open $dir $!\n";
     $sites->insert(map { unpack 'a4',$_ } grep { /\.${yy}d\.(Z|bz2)$/ } readdir($dh));
   }
   return $sites
}
sub get_rinex {
   my $date = shift;
   my $sites = shift;

   my ($yyyy,$doy) = split(' ',get_date($date,"doy"));
   my $cal = get_date($date,'cal');
   my $gwd = get_date($date,'gwd');
      $doy=sprintf("%03d",$doy);
   my $yy = unpack("x2a2",$yyyy);

   my $here = cwd();

   my $cache_dir;
   if ($archive) {
       $cache_dir = "$par{RNX_DIR_ARCHIVE}";
   } else {
       $cache_dir = "$par{RNX_DIR}";
   }
   if ( ! -d "$cache_dir/$yyyy/$doy" ) {
     mkpath "$cache_dir/$yyyy/$doy";
   }
   chdir "$cache_dir/$yyyy/$doy";

   #print cwd(),"\n";

  my @info =  ( 
       { host => "data-out.unavco.org", 
	     dir  => "/pub/rinex/obs/$yyyy/$doy",
         name  => 'unavco'},
	   { host => 'ftp.panga.cwu.edu',
	     dir  => "/pub/data/$yyyy/$doy/30sec",
         name  => 'panga'},
 #{ host => 'ftp.panga.cwu.edu',
#	     dir  => "/pub/data/local_nets/$yyyy/$doy/30sec",
 #        name  => 'panga'},
    #  { host => "data-out2.unavco.org", 
	#    dir  => "/pub/rinex/obs/$yyyy/$doy",
    #    name  => 'unavco-mirror'},
	   { host => "cddis.gsfc.nasa.gov", 
	     dir  => "/pub/gps/data/daily/$yyyy/$doy/${yy}d",
         name  => 'cddis'},
	   { host => "garner.ucsd.edu",
	     dir  => "/pub/rinex/$yyyy/$doy",
         name  => 'sopac'},
	   { host => 'cors.ngs.noaa.gov',
	     dir  => "/cors/rinex/$yyyy/$doy",
         name  => 'cors'},
	 );

  if ( $proj eq 'nepal') {
          push @info,{  host => 'ftp.panga.cwu.edu',
                   dir  => "/pub/data/nepal/$yyyy/$doy/30sec",
                   name  => 'panga'};
     push @info,{  host => 'ftp.panga.cwu.edu',
                   dir  => "/pub/data/incoming/NEPAL_CALTECH/$yyyy/$doy",
                   name  => 'panga'};
     #get_nepal_list($yyyy,$doy);
  }

  my $downloaded = Set::Scalar->new();
  my $archived = archived($yyyy,$doy);
  $sites = $sites->difference($archived);

  print "Attempting to download ",$sites->size()," RINEX files\n";
  foreach my $src ( @info ) {	  
      my $just_downloaded = get_rinex_from_source($src,$date,$sites);
      print "Downloaded ",$just_downloaded->size()," Sites from ",(uc $src->{'name'}),"\n";
      $sites = $sites->difference($just_downloaded);
      $downloaded->insert($just_downloaded->elements());
      print $sites->size()," Sites left to download\n";
  } #end info

  my $i=1;
  my $nd = $downloaded->size();
  my $summary =<<EOT;
#=========================================== 
Summary RINEX Download : $cal ; YYYY,DOY = $yyyy,$doy ; WWWWD = $gwd
Downloaded                    : $nd
#===========================================
EOT

 print $summary;

}

sub repack_rnx {
   my $rinex = shift;

   $rinex = basename($rinex);
   my ($rname,$rext,$cext) = split /\./, $rinex ; 		#get parts of filename
      $rname =~ s/.$/0/;
   my $rnx = "$rname.$rext";
   my $unzip = $cext eq 'bz2' ? 'bzip2 -dc': 'gzip -dc';
   system("$unzip $rinex 1> $rnx ");
   if ( $rext =~ /\d\d[oO]$/ ) {
      $rext =~ s/o$/d/i;
      system("rnx2crz $rnx");
   }
   else {
     `compress -f $rnx`;
   }
   return "$rnx.Z" 
}

sub get_nepal_list {
 my $yyyy = shift;
 my $yy = unpack 'x2a2',$yyyy;
 my $doy  = shift;
 my $url_base = "http://jishin.gps.caltech.edu/gps/nepal/rinex";
 my @rnx = map { /href="([a-z_0-9]{4}\d{3}.*?)">/g  } 
           get("$url_base/$yyyy/$doy");
 foreach my $r ( @rnx ) {
   next if -e $r;
   my $s = unpack 'a4',$r;
   get_nepal_caltech_rnx($url_base,$r);
   if ( $r ne "$s${doy}0.${yy}d.Z" )
   {
     repack_rnx($r);
     unlink $r;
   }
 }

}
sub get_nepal_caltech_rnx {
  my $url_base = shift;
  my $rnx      = shift;
  open(my $fh,">", $rnx ) or die "Couldn't open $rnx for writing $!\n";
  print $fh get("$url_base/$rnx");
  close $fh;
}

sub get_rinex_from_source {
    my $src = shift;
    my $date = shift;
    my $sites = shift;

    my ($yyyy,$doy) = split(' ',get_date($date,"doy"));
    my $yy = unpack 'x2a2',$yyyy;

    my (@list,@rnx)=();
    my $host  = $src->{host}; 
    my $dir   = $src->{dir}; 
    my $ag    = uc $src->{name}; 
    my $password = 'gnet@geology.cwu.edu';

    print "Connecting to $ag ( $host )";
    my $ftp = Net::FTP->new( "$host",
	                     Timeout => 30,
		             Debug => 0,
		             Passive => 1 ) or print ( "Can't connect to host $host: $@\n" );
    if ( ! defined $ftp || ! $ftp ) {
     print " FAILED\n";
     next;
    }
    else {
     print " SUCCESS\n";
    }
    $ftp -> login ( "anonymous", $password ) 
		or print  ( " Couldn't login into $host\n");
    $ftp -> binary () 
	   or print  ( " Couldn't set type to binary in $host\n");
    $ftp -> cwd ( $dir ) 
	   or print  ( " Couldn't change to dir $dir\n");
    
    if ( uc($ag) eq 'CORS' ) {
       my $dirinfo_hashref = $ftp->dir_hashref;
       foreach my $remotedir (keys %$dirinfo_hashref) {
           my $remotefile = "$remotedir/$remotedir${doy}0.${yy}d.Z";
           push(@rnx,$remotefile)
       }
    } elsif ( $ag eq 'PANGA' ) {
  	  @rnx = grep { /^.{4}$doy[a-z0]\.\d{2}(d|o)\.(bz2|gz|Z|zip)$/i } $ftp->ls("*.*");
    } else {
  	  @rnx = grep { /d\.Z$/ } $ftp->ls("*.*");
    }

    my %files=();
    foreach my $r ( @rnx ) {  ### Searching $host [===|    ] % done
      my $rnx = basename($r);
      my $s =unpack("a4", $rnx);
      my ($rnx_local) = glob("$s*");
      if ( $sites->has($s) && ! $rnx_local ) {
        $files{$s} = $r;
      }
    }

    my $ok;
    my $downloaded = Set::Scalar->new();
    foreach my $s ( sort keys %files ) { ### Downloading from $host [===|   ] % done 
	# Skip this file if it already exists
	if ( -e basename($files{$s})) {
           print basename($files{$s})," already exists\n";
	   next;
        }
	#next if -e basename($files{$s});
	# Download the file, OK is undef if this fails
	print "Downloading ", $files{$s},"\n";
        $ok=$ftp->get($files{$s});

	# Repack RINEX files so that they are uniformly named
        if ( $files{$s} ne "$s${doy}0.${yy}d.Z" ) {
            my $old_rnx = $files{$s};
            $files{$s}=repack_rnx($files{$s});
            unlink $old_rnx;
        }

	# 
        if ( defined $ok ) {
	    $downloaded->insert($s);
        } else {
            print 'couldn\'t get ', basename($files{$s}),
              'Msg:',$ftp->message; 
        } 
    }

   $ftp -> quit() 
        or print  ( " Couldn't quit from $host\n");

   return $downloaded;
}
