This is the mail archive of the
cygwin@cygwin.com
mailing list for the Cygwin project.
Re: Using a real mirroring tool...
Hi!
Charles Wilson wrote:
> A few notes:
>
> ...
>
> rsync can't merge.
Yes, but Michael A. Chases clean_setup.pl can! I allowed myself to add a
new option to clean_setup.pl, it exports the list of missing files to a
file. Then you can use wget to get all the missing files.
I attached the modified clean_setup.pl plus a shell scrip to get the
missing files from a mirror. You have to modify the getmissing.sh to
choose your mirror and your target download directory. The directory and
both files must exist the same directory as your setup.exe. Just a quick
hack, I guarantee nothing!
Bye
Volker
#!/bin/bash
# Mirror script for cygwin
# Uses Michael A. Chase "cleanup_setup.pl"
# in a modified version (V1.0303)
# and wget
# getmissing.sh and cleanup_setup.pl must reside in the same directory
# as setup.exe
# Files to download
filelist="missing.lst"
# Choose your mirror with directory of setup.ini
mirror="ftp://ftp.inf.tu-dresden.de/software/windows/cygwin32/";
# Cut directories 1 2 3
cutdir=3
# Name your download directory
mirrordir="_mymirror"
wget --mirror --no-host-directories --cut-dirs=$cutdir --passive-ftp -P $mirrordir ${mirror}setup.ini
./clean_setup.pl -Arch -writelist
wget --mirror --no-host-directories --cut-dirs=$cutdir --passive-ftp -P $mirrordir -i $filelist -B $mirror
./clean_setup.pl -Setup
#! /usr/bin/perl -w
require 5.005;
# Delete old files from Cygwin setup.exe archive subdirectories
# Name of file: cleanup_setup.pl
#
# Type of file: Perl script
#
# Author: Michael A. Chase
#
# Purpose: Clean out setup.exe archive subdirectories.
# Syntax: See $HelpText Below
#-------------------------- MODIFICATION HISTORY -----------------------------
$VERSION = '1.0303';
# 020430 V. Quetschke Add -writelist
# Writes list of missing files to missing.lst.
# 020423 M. Chase Add -source, -install, -H options.
# Removed directory names from Missing Files list.
# $VERSION = '1.0301';
# 020417 M. Chase Stop lc()ing file names during setup.ini parsing.
# $VERSION = '1.0300';
# 020416 M. Chase Ignore setup.ini while collecting archive file names.
# Remove directories in reverse length order.
# $VERSION = '1.0202';
# 020416 M. Chase Properly ignore directories in obsolete file search.
# $VERSION = '1.0201';
# 020404 M. Chase Ignore files found more than once in move loop.
# Protect against attempts to move files from different
# trees over each other in the base tree.
# $VERSION = '1.0200';
# 020326 M. Chase Properly handle directory case.
# Optionally move archives to base directory tree.
# $VERSION = '1.0100';
# 020212 M. Chase Handle multiple setup.ini files and file sources.
# $VERSION = '1.0000';
# 010422 M. Chase First draft.
use FindBin qw( $RealBin $RealScript );
use File::Basename qw( &basename &dirname &fileparse );
use File::Find;
use File::Spec;
use File::Path qw( &mkpath );
use File::Copy qw( © &move );
use Getopt::Long;
use strict;
use integer;
use vars qw( $VERSION );
# Initialize options
my $bArch = 0;
my $bMove = 0;
my $bSetup = 0;
my $bCurr = 1;
my $bPrev = 1;
my $bTest = 1;
my $bExp = 1;
my $bInstall = 1;
my $bSource = 0;
my $bWrite = 0;
my @sHide = ();
my @sIgnore = ();
my $sDir0 = File::Spec -> rel2abs( "." );
# Syntax description
sub usage {
my ( $sOpt, $sVal, @sMsg ) = @_;
my $sHelpText = <<END_HELP_TEXT;
Cleanup Cygwin setup.exe package cache directories
syntax: $RealScript [opt]
Opt: ($VERSION)
-[no]Move = [Don't] Move archive files to base directory tree ($bMove)
-[no]Arch = [Don't] Delete obsolete archives and directories ($bArch)
-[no]Setup = [Don't] Delete obsolete setup.ini files ($bSetup), forces -Move
-[no]curr = [Don't] Report missing [curr] files ($bCurr)
-[no]prev = [Don't] Report missing [prev] files ($bPrev)
-[no]test = [Don't] Report missing [test] files ($bTest)
-[no]exp = [Don't] Report missing [exp] files ($bExp)
-[no]install = [Don't] Report missing install archives ($bInstall)
-[no]source = [Don't] Report missing source archives ($bSource)
-writelist = Write missing files report (missing.lst) to disk ($bWrite)
-base dir = Archive cache base directory ($sDir0)
-I mask = Ignore files and directories that match mask, multiple allowed
-H mask = Ignore packages that match mask, multiple allowed
mask is a filename wildcard mask, not a regular expression
Arg:
END_HELP_TEXT
# Balance quotes in here document # ' # "
my $nRet = 'help' eq $sOpt ? 0 : 0 + $sVal;
select STDERR if $nRet;
foreach ( @sMsg, $sHelpText ) { s/\s+$//; print "$_\n"; }
exit $nRet;
}
# Parse command line
Getopt::Long::config( qw( no_ignore_case no_auto_abbrev require_order ) );
GetOptions(
'Arch!' => \$bArch,
'Move!' => \$bMove,
'Setup!' => \$bSetup,
'curr!' => \$bCurr,
'prev!' => \$bPrev,
'test!' => \$bTest,
'exp!' => \$bExp,
'install!' => \$bInstall,
'source!' => \$bSource,
'writelist!' => \$bWrite,
'base=s' => \$sDir0,
'Hide|H=s@' => \@sHide,
'Ignore|I=s@' => \@sIgnore,
'help|h' => \&usage ) or usage( 'die', 1 );
$bMove ||= $bSetup; # Don't remove setup files unless also moving archves
chdir $sDir0 or usage( 'die', 1, "Can't change directory to $sDir0, $!\n" );
# Provide default values for unset options and parameters
# Report arguments
$sDir0 = File::Spec -> rel2abs( "." );
$sDir0 =~ s,\\,/,g;
my $sOpt = '';
$sOpt .= "\nMoving archives to base directory tree" if $bMove;
$sOpt .= "\nDeleting unused files and empty directores" if $bArch;
$sOpt .= "\nDeleting obsolete setup.ini files" if $bSetup;
$sOpt .= "\nIgnoring files and directories: " . join ' ', map { "'$_'" }
@sIgnore if @sIgnore;
$sOpt .= "\nIgnoring packages: " . join ' ', map { "'$_'" } @sHide if @sHide;
print <<HERE;
Base Directory: $sDir0$sOpt
HERE
# Build file or directory name matcher
# Adapted from Recipe 6.10 in Perl Cookbook
sub rfMatch {
my ( $bDefault, $op, @sMask ) = @_; # @sMask must be a lexical array
return sub { return $bDefault; } if 3 > @_;
my $sExpr = join " $op\n",
map {
# Convert file expansion mask to regular expression
$sMask[$_] =~ s/\./\\./g;
$sMask[$_] =~ s/\?/.?/g;
$sMask[$_] =~ s/\*/.*/g;
"m:^\$sMask[$_]\$:oi";
} 0 .. $#sMask;
my $rfMatch = eval "sub { local \$_ = shift;\nreturn $sExpr; };";
die $@ if $@;
return $rfMatch;
}
local *bHide = rfMatch( 0, '||', @sHide );
local *bIgnore = rfMatch( 0, '||', @sIgnore );
# Find and parse setup.ini files, collect other filenames at the same time
my ( %aSetup, $sRel, $sName, %sTarBall, %bDir );
my $wanted = sub {
# Skip ., .., and files and directories in ignore list
if ( '.' eq $_ || '..' eq $_ ) {
$File::Find::prune = 1 if '..' eq $_;
return;
}
if ( bIgnore( $_ ) ) {
$File::Find::prune = 1 if -d $_; # Prune if a directory
return;
}
# Handle directory or file
$sRel = sRel( $File::Find::dir, $sDir0 );
if ( -d $_ ) { $bDir{sRel( $File::Find::name, $sDir0 )} = 1; }
# Remember directory name for possible removal
elsif ( "setup.ini" eq $_ ) {
# Parse setup.ini
my @aSetup;
# Get list of files to leave alone, includes subdirectory path
# setup-timestamp: 1012849221
# install: latest/bash/bash-2.05-1.tar.gz 576828
# source: latest/bash/bash-2.05-1-src.tar.gz 1792319
my ( $bHide, $sGroup, $sType, $sFile, $sSize, $sVol, $sSubDir, $sName );
my $sSetup = $File::Find::name;
open( SETUP, $sSetup ) or usage( 'die', 1, "Can't open $sSetup, $!" );
while ( <SETUP> ) {
( $sType, $sFile, $sSize ) = split /\s+/, $_;
if ( 'setup-timestamp:' eq $sType ) {
$aSetup[0] = $sFile; # Actually timestamp
}
elsif ( 'install:' eq $sType ) {
next if $bHide;
( $sVol, $sSubDir, $sName ) = File::Spec -> splitpath( $sFile );
$sSubDir = File::Spec -> canonpath( $sSubDir );
$aSetup[1]{$sName} = [ $sGroup, $sSize, $sSubDir ];
}
elsif ( 'source:' eq $sType ) {
next if $bHide;
( $sVol, $sSubDir, $sName ) = File::Spec -> splitpath( $sFile );
$sSubDir = File::Spec -> canonpath( $sSubDir );
$aSetup[2]{$sName} = [ $sGroup, $sSize, $sSubDir ];
}
elsif ( '[' eq substr( $_, 0, 1 ) ) { s/\s+$//; $sGroup = $_; } # ]
elsif ( s/^@\s+// ) {
s/\s+$//;
$bHide = bHide( $_ );
$sGroup = '[curr]';
}
}
close SETUP;
usage( 'die', 1, "Nothing found in $sSetup" )
if 3 != @aSetup || ! $aSetup[0];
$aSetup{$sRel} = \@aSetup;
}
elsif ( ".tar.bz2" eq substr( $_, -8 ) || ".tar.gz" eq substr( $_, -7 ) ) {
# Save name of archive file
$sTarBall{$_}{File::Spec -> canonpath( $sRel )} = -s $_;
}
# Currently ignoring other types of files
};
find( $wanted, $sDir0 );
usage( "die", 1, "No setup.ini files found" ) if ! %aSetup;
# Pick newest setup.ini
my ( $tNewest, $sNewest, @sOldSetup ) = ( 0, "" );
foreach ( sort keys %aSetup ) {
if ( $tNewest < $aSetup{$_}[0] ) {
$tNewest = $aSetup{$_}[0];
$sNewest = $_;
}
}
print "Newest setup.ini files: ", scalar gmtime( $tNewest ), " GMT\n";
foreach ( sort keys %aSetup ) {
print " ", sUnPercent( sRel( $_ ) ), "\n" if $tNewest == $aSetup{$_}[0];
}
# Report or remove obsolete setup.ini files
foreach ( sort keys %aSetup ) {
push @sOldSetup, $_ if $tNewest > $aSetup{$_}[0];
}
print "Obsolete setup.ini files:\n" if @sOldSetup;
foreach ( @sOldSetup ) {
print " ", scalar gmtime( $aSetup{$_}[0] ), " GMT ",
sUnPercent( sRel( $_ ) ), "\n";
unlink "$_/setup.ini" or print " *** Can't remove, $!" if $bSetup;
}
# Check found files against those listed in latest setup.ini
my ( @sDir, $sDir, $sFile, @sDup, %sMove, @sRemove, @sWrongSize );
my %aInstall = ( %{$aSetup{$sNewest}[1]} );
my %aSource = ( %{$aSetup{$sNewest}[2]} );
foreach $sName ( sort keys %sTarBall ) {
@sDir = sort keys %{$sTarBall{$sName}};
$sDir = $sDir[0];
$sFile = File::Spec -> catfile( $sDir, $sName );
if ( 1 < @sDir ) {
# More than one copy of a file was found
push @sDup, "$sName: " . join( ", ", map { sUnPercent( $_ ) } @sDir );
}
elsif ( exists $aInstall{$sName} ) {
if ( $sDir eq $aInstall{$sName}[2] ) {} # Already in right place
elsif ( $sTarBall{$sName}{$sDir} != $aInstall{$sName}[1] ) {
# Wrong size
push @sWrongSize,
"$sFile: $sTarBall{$sName}{$sDir} != $aInstall{$sName}[1] in " .
sUnPercent( $sDir );
}
else {
$sMove{$sFile} = File::Spec -> catfile( $aInstall{$sName}[2], $sName );
}
}
elsif ( exists $aSource{$sName} ) {
if ( $sDir eq $aSource{$sName}[2] ) {} # Already in right place
elsif ( $sTarBall{$sName}{$sDir} != $aSource{$sName}[1] ) {
# Wrong size
push @sWrongSize,
"$sFile: $sTarBall{$sName}{$sDir} != $aSource{$sName}[1] in " .
sUnPercent( $sDir );
}
else {
$sMove{$sFile} = File::Spec -> catfile( $aSource{$sName}[2], $sName );
}
}
else {
# File not in setup.ini
push @sRemove, $sFile;
}
}
# Check for missing files
my @sMissing = ();
my @sMissingList = ();
if ( $bInstall ) {
foreach ( keys %aInstall ) {
if ( ! exists $sTarBall{$_} &&
( $bCurr || '[curr]' ne $aInstall{$_}[0] ) &&
( $bPrev || '[prev]' ne $aInstall{$_}[0] ) &&
( $bTest || '[test]' ne $aInstall{$_}[0] ) &&
( $bExp || '[exp]' ne $aInstall{$_}[0] ) ) {
push @sMissing, "$aInstall{$_}[0] $aInstall{$_}[2]/$_" ;
# Alternative: $aInstall{$_}[0] . " " .
# File::Spec -> catfile( $aInstall{$_}[2], $_ ) }
push @sMissingList, "$aInstall{$_}[2]/$_" ;
}
}
}
if ( $bSource ) {
foreach ( keys %aSource ) {
if ( ! exists $sTarBall{$_} &&
( $bCurr || '[curr]' ne $aSource{$_}[0] ) &&
( $bPrev || '[prev]' ne $aSource{$_}[0] ) &&
( $bTest || '[test]' ne $aSource{$_}[0] ) &&
( $bExp || '[exp]' ne $aSource{$_}[0] ) ) {
push @sMissing, "$aSource{$_}[0] $_" ;
# Alternative: $aSource{$_}[0] . " " .
# File::Spec -> catfile( $aSource{$_}[2], $_ ) }
push @sMissingList, "$aSource{$_}[2]/$_" ;
}
}
}
@sMissing = sort @sMissing;
# Complain about errors
print join( "\n ", "\nDuplicate Files", @sDup ), "\n" if @sDup;
print join( "\n ", "\nWrong Size Files", @sWrongSize ), "\n" if @sWrongSize;
print join( "\n ", "\nMissing Files:", @sMissing ), "\n" if @sMissing;
# Report list of missing files to disk
if ( $bWrite ) {
@sMissingList = sort @sMissingList;
open(REPORT, '>', 'missing.lst');
print( REPORT join( "\n", @sMissingList ),"\n" ) if @sMissingList;
close(REPORT);
}
# Move queued files to base directory tree
if ( %sMove ) {
print $bMove ? "\n" : "\nNot ", "Moving files to base directory tree\n";
my ( $sFrom, $sTo );
foreach $sFrom ( sort keys %sMove ) {
$sTo = $sMove{$sFrom};
$sDir = dirname( $sTo );
print sUnPercent( " $sFrom -> $sDir\n" );
if ( -e $sTo ) { print " *** Target already exists\n"; }
elsif ( $bMove ) {
mkpath( $sDir ) if ! -d $sDir;
move( $sFrom, $sTo ) or print " *** Can't move, $!\n";
}
}
}
# Remove files not listed in setup.ini
if ( @sRemove ) {
print $bArch ? "\n" : "\nNot ", "Removing files not in setup.ini\n";
foreach $sFile ( @sRemove ) {
print sUnPercent( " $sFile\n" );
if ( $bArch ) {
unlink( $sFile ) or print " *** Can't remove, $!\n";
}
}
}
# Remove empty directories
my ( @g, $bFound );
foreach $sRel ( sort { length $b <=> length $a || $a cmp $b } keys %bDir ) {
@g = glob( File::Spec -> catfile( $sRel, "*" ) );
if ( ! @g ) {
print $bArch ? "\n" : "\nNot ", "Removing Empty Directories\n"
if ! $bFound++;
print sUnPercent( " $sRel\n" );
if ( $bArch ) {
rmdir $sRel or print " *** Can't rmdir, $!\n";
}
}
}
# Copy latest setup.ini to base directory
if ( "." ne $sNewest ) {
print sUnPercent( "\nCopying setup.ini from $sNewest to $sDir0\n" );
copy( "$sNewest/setup.ini", "$sDir0/setup.ini" )
or print " *** Can't copy setup.ini, $!\n";
}
exit 0;
# Produce relative file or directory
sub sRel {
my ( $sAbs, $sBase ) = @_;
$sBase = "." if ! defined $sBase || ! length $sBase;
my $sRel = File::Spec -> abs2rel( $sAbs, $sBase );
$sRel =~ s,\\,/,g;
$sRel =~ s/^\w://;
$sRel = "." if ! length $sRel;
return $sRel;
}
# Convert %xx to characters
sub sUnPercent {
local ( $_ ) = @_;
s/\%([0-9a-f]{2})/chr(hex($1))/gie;
return $_;
}
--
Unsubscribe info: http://cygwin.com/ml/#unsubscribe-simple
Bug reporting: http://cygwin.com/bugs.html
Documentation: http://cygwin.com/docs.html
FAQ: http://cygwin.com/faq/