Re: FYI, cvs-to-git mirroring hiccup: just resolved

[Jim Meyering <jim at meyering dot net>]

mikhail.terekhov@emc.com wrote:
>> That was because my cvs-to-git mirroring script was using
>> /cvs/src/src/gdb/gdbinit.in,v (a witness that the configuration
>> repo and modules are well-configured), yet that file has recently
>> been cvs-removed.
>>
>> I've changed the mirroring config to use gdb-demangle.h,v instead,
>> and am running the sync manually now to be sure.
>
> Jim,
>
> Is this cvs-to-git mirroring script available somewhere to learn from?

There are two scripts.  A driver I call mirror-sw (sw=sourceware, to
contrast with another variant I've used for savannah-based projects)
and another that it invokes to do the bulk of the work, called
mirror-cvs-to-git.

You may not want to try to learn from these scripts.
They're not well documented.

But for the record, here they are:

#!/bin/sh
# sync a CVS-to-GIT mirror on sourceware.org
# Take a package name and map that to the parameters needed to invoke
# mirror-cvs-to-git.  Then, push any changes from the temporary git
# repository to the destination one.

PATH=$HOME/bin:/usr/local/bin:/sbin:/usr/sbin:/bin:/usr/bin
export PATH

ME=$(basename "$0")
die() { echo >&2 "$ME: $@"; exit 1; }
filter_stderr()
{
  # Usage: filter-stderr EGREP_REGEXP 'command'
  regex=$1 cmd=$2
  (exec 3>&1; eval "$cmd" 2>&1 >&3 | grep -E -v "$regex" 1>&2)
}

verbose=
case $# in
  0) ;;
  *) case $1 in --verbose) verbose=--verbose; shift;; esac;;
esac

binutils_exclude='
blt
cgen
contrib
dejagnu
expat
expect
gdb
itcl
iwidgets
libdecnumber
libgloss
libgui
mmalloc
newlib
proto-toplev
rda
readline
sid
sim
tcl
tix
tk
utils
winsup
'

gdb_exclude='
binutils
blt
cgen
contrib
dejagnu
elfcpp
expat
expect
gas
gdb/gdbtk
gold
gprof
itcl
iwidgets
ld
libgloss
libgui
mmalloc
newlib
proto-toplev
rda
sid
tcl
tix
tk
utils
winsup
'

newlib_exclude='
bfd
binutils
blt
cgen
cpu
dejagnu
elfcpp
expat
expect
gas
gdb
gold
gprof
include
intl
itcl
iwidgets
ld
libdecnumber
libgui
libiberty
mmalloc
opcodes
proto-toplev
rda
readline
sid
sim
tcl
tix
tk
utils
winsup
'

repo=$1
case $repo in
  gdb)
      exclude=$gdb_exclude
      # CAUTION: m=
      # Note how here, m is not set, yet it is for binutils and newlib.
      cvs_mod=src url=/cvs/src f=gdb/gdb-demangle.h,v g=/git/gdb.git;;

  binutils)
      exclude=$binutils_exclude
      m=binutils
      cvs_mod=src url=/cvs/src f=binutils/arlex.l,v g=/git/binutils.git;;

  cluster)
      cvs_mod=cluster url=/cvs/cluster f=gfs/gfs_mkfs/mkfs_gfs.h,v
                                                      g=/git/cluster.git;;

  newlib)
    exclude=$newlib_exclude
    m=newlib
    cvs_mod=src url=/cvs/src f=newlib/testsuite/lib/newlib.exp,v
     g=/git/newlib.git;;

  automake)
    cvs_mod=automake url=/cvs/automake f=lib/Automake/RuleDef.pm,v
                                                      g=/git/automake.git;;
  dm|device-mapper)
    cvs_mod=device-mapper url=/cvs/dm f=dmsetup/dmsetup.c,v g=/git/dm.git ;;
  lvm|lvm2) cvs_mod=LVM2 url=/cvs/lvm2 f=tools/lvm.c,v      g=/git/lvm2.git ;;

  libc) cvs_mod=libc url=sourceware.org::glibc-cvs f=manual/libc.texinfo,v
                                                      g=/git/glibc.git ;;

  glibc-ports) cvs_mod=ports url=sourceware.org::glibc-cvs f=bare/brdinit.c,v
                                                      g=/git/glibc-ports.git ;;

  *) die "Usage: $ME package_name" ;;
esac
exclude=$(printf %s "$exclude"|tr '\012' ' '|sed 's/ $//')

test -z "$m" && m=$cvs_mod

tmp_repo=$HOME/mirror-git-to-cvs/repo/$m

mirror_cmd="mirror-cvs-to-git     \
  $verbose            \
  --exclude='$exclude' \
  --rsync-url=$url    \
  --module=$cvs_mod         \
  --git-dir=$tmp_repo \
  --state-dir=$HOME/mirror-git-to-cvs/.state/$m \
  --key-file=$f       \
  --user-map=$HOME/mirror-git-to-cvs/git-user-map/$m"

mirror_re=$(printf %s '
no patchset for tag raeburn
|Skipping #CVSPS_NO_BRANCH
|branch #CVSPS_NO_BRANCH not found
|\* UNKNOWN LINE \* Branches:
|revision .* is tagged but not present
' |tr -d '\012')

filter_stderr "$mirror_re" "$mirror_cmd"

# Push unconditionally.  Otherwise, at least the initial
# run fails to push.
# case $? in
#   0) exit 0 ;; # nothing rsync'd, no push required
#   1)        ;; # fall through
#   *) exit 1 ;;
# esac

push_re=$(printf %s '
Everything up-to-date
|^To /git/\S+\.git$
|^   \x{7}\.\.\x{7}  \S+ -> \S+$
|^updating .refs/heads/master.$
|^Branch \S+ already exists\.$
|^Removing duplicate objects:.*done\.$
|^Delta compression using up to 4 threads\.$
|^Generating pack\.\.\.$
|^Done counting \d+ objects\.$
|^Deltifying \d+ objects\.\.\.$
|^Total \d+ \(delta \d+\),
|^refs/heads/master: \x{40} -> \x{40}$
|^ +\d+% \(1/\d+\) done
|^  from \x{40}$
|^  to   \x{40}$
|((Compress|Count|Writ)ing objects:)
' |tr -d '\012' \
  |sed 's/\\S/[^[:space:]]/g;s/\\d/[0-9]/g;s/\\x/[[:xdigit:]]/g')

case $repo in
  libc|glibc-ports) spec='refs/heads/*:refs/heads/cvs/*';;
  *)    spec='refs/heads/*:refs/heads/*';;
esac

push_cmd="GIT_DIR=$tmp_repo/.git git push --force --tags $g +'$spec'"

# Push to the public git repo.
filter_stderr "$push_re" "$push_cmd"

exit 0

#!/bin/sh
# Sync a local git repository from a remote CVS repository.

VERSION='2012-04-18 15:36' # UTC
# The definition above must lie within the first 8 lines in order
# for the Emacs time-stamp write hook (at end) to update it.
# If you change this file with Emacs, please let the write hook
# do its job.  Otherwise, update this string manually.

# Requirements:
# - git and cvs, of course
# - cvsps (used by git cvsimport)
# - GNU find and xargs

# FIXME: Add an option to handle *local* master CVS repo,
# hence no need for the rsync run: just use original

# FIXME: while the rsync copies any write lock files, sleep+repeat-up-to-N-times
# Hmm... if a modified file is so new that it'd trigger the "too recent"
# skip-changeset, then consider sleeping.  But then we'd need to know
# the cron-job frequency, or take out a lock.

# Hmm... better (simpler), just remove the write-lock files.
# That's ok, since the offending, partial commit, should be classified
# as "too recent", and deferred.

# FIXME: so, to implement the above, rsync-exclude any in-tree write-lock files.

# FIXME: Document that user-map file is required only on the first iteration

ME=$(basename "$0")
die() { echo >&2 "$ME: $@"; exit 2; }

# Return 0 if the mtime of $FILE is at least $N_MINUTES in the past.
timestamp_old_enough()
{
  case $# in
    2) ;;
    *) die 'Usage: timestamp_old_enough file n_minutes';;
  esac

  case $2 in
    [0-9]*) ;;
    *) die 'Usage: timestamp_old_enough file n_minutes';;
  esac
  file=$1
  n=$2
  old_enough=$(find "$file" -mmin +$n)
  case $old_enough in
    '') return 1;;
    *) return 0;;
  esac
}

move_if_change()
{
  case $# in
  2) ;;
  *) eval echo >&2 'Usage: move_if_change file_1 file_2'; exit 2;;
  esac

  if [ -r $2 ] ; then
    if cmp $1 $2 > /dev/null ; then
      : # echo $2 is unchanged
      rm $1
    else
      mv $1 $2
    fi
  else
    mv $1 $2
  fi
}

usage ()
{
  echo "\
Usage: $ME \\
  --rsync-url=example.org::dir/... \\
  --module=module_name \\
  --git-dir=DIR \\
  --state-dir=DIR \\
  [OPTION...]

Synchronize a git repository mirror from a cvs repository.

Options:

  --rsync-url=RSYNC_URL
             Specify an rsync URL, e.g., example.org::repo.
	     You'd rsync this URL to get the entire CVS repository,
	     often including a top-level CVSROOT/ directory.

  --exclude=SPC_SEPARATED_LIST
             Specify names to exclude.

  --module=CVS_MODULE
             Specify the CVS module name to mirror into a git repository.

  --state-dir=DIR
             Specify a directory, DIR, in which to store sufficient
	     state so that subsequent runs are efficient.  Things we
	     store in there: rsync'd CVS repo, the user-map file.
	     FIXME: currently we create DIR if it doesn't exist,
	     but only using "mkdir", not "mkdir -p".

  --git-dir=DIR
             Specify a directory, DIR, in which the resulting .git
	     directory resides (or will reside, if it doesn't yet exist).
	     FIXME: currently we create DIR if it doesn't exist,
	     but only using "mkdir", not "mkdir -p".

  --key-file=FILE
             Specify a file (usually ending in ,v) in the desired repo.
             FIXME: give an example.

  --user-map=FILE
             Specify the mapping of user names in CVS logs to the
	     (User Name, <user@email.dom>) pairs git uses.  Each line
	     must look like \"username=User Name <user@email.dom>\".
	     Specify this file so that git tools can display the real
	     name and email address of each change-set committer.

  --help     display this help and exit
  --version  output version information and exit

Exit status:
  0 rsync pulled in _no_ changes
  1 rsync *did* pull in changes
  2 abnormal (error) termination


EXAMPLE

  m=device-mapper url=sourceware.org::dm-cvs f=dmsetup/dmsetup.c,v
  mirror-cvs-to-git \\
    --rsync-url=\$url \\
    --module=\$m \\
    --git-dir=/work/co/git-repo/\$m \\
    --state-dir=/work/co/git-repo/.state/\$m \\
    --key-file=\$f \\
    --user-map=\$HOME/tmp/misc/git-user-map/\$m

  m=LVM2 url=sourceware.org::lvm2-cvs f=tools/lvm.c,v
  m=emacs url=cvs.sv.gnu.org::sources/emacs f=src/emacs.c,v
  m=gnulib url=cvs.sv.gnu.org::sources/gnulib f=doc/gnulib.texi,v

  Report bugs to <jim@meyering.net>."
}

Exit ()
{
  (exit $1); exit $1
}

rsync_url=
module=
exclude=
git_dir=
state_dir=
user_map=

verbose=

{
  while test $# -gt 0; do
    case "$1" in

      --module )
        shift
        if test $# = 0; then
          die "missing argument for --module"
        fi
        module="$1"
        shift ;;
      --module=* )
        module=`echo "X$1" | sed -e 's/^X--module=//'`
        shift ;;

      --exclude )
        shift
        if test $# = 0; then
          die "missing argument for --exclude"
        fi
        exclude="$1"
        shift ;;
      --exclude=* )
        exclude=`echo "X$1" | sed -e 's/^X--exclude=//'`
        shift ;;

      --git-dir )
        shift
        if test $# = 0; then
          die "missing argument for --git-dir"
        fi
        git_dir="$1"
        shift ;;
      --git-dir=* )
        git_dir=`echo "X$1" | sed -e 's/^X--git-dir=//'`
        shift ;;

      --key-file )
        shift
        if test $# = 0; then
          die "missing argument for --key-file"
        fi
        key_file="$1"
        shift ;;
      --key-file=* )
        key_file=`echo "X$1" | sed -e 's/^X--key-file=//'`
        shift ;;

      --user-map )
        shift
        if test $# = 0; then
          die "missing argument for --user-map"
        fi
        user_map="$1"
        shift ;;
      --user-map=* )
        user_map=`echo "X$1" | sed -e 's/^X--user-map=//'`
        shift ;;

      --state-dir )
        shift
        if test $# = 0; then
          die "missing argument for --state-dir"
        fi
        state_dir="$1"
        shift ;;
      --state-dir=* )
        state_dir=`echo "X$1" | sed -e 's/^X--state-dir=//'`
        shift ;;

      --rsync-url )
        shift
        if test $# = 0; then
          die "missing argument for --rsync-url"
        fi
        rsync_url="$1"
        shift ;;
      --rsync-url=* )
        rsync_url=`echo "X$1" | sed -e 's/^X--rsync-url=//'`
        shift ;;

      --verbose | --verbos | --verbo | --verb )
        verbose=-v
        shift ;;
      --help | --hel | --he | --h )
        usage
        Exit $? ;;
      --version | --versio | --versi | --vers )
        echo "$ME version $VERSION"
        Exit $? ;;
      -- )
        # Stop option processing
        shift
        break ;;
      -* )
        echo "$ME: unknown option $1" 1>&2
        echo "Try '$ME --help' for more information." 1>&2
        Exit 1 ;;
      * )
        break ;;
    esac
  done
}

ok=no
test -n "$rsync_url" &&
  test -n "$module" &&
  test -n "$git_dir" &&
  test -n "$key_file" &&
  test -n "$state_dir" &&
  ok=yes
test $ok = yes || die "required argument not specified"

#############################################################
test -d "$git_dir" \
  || { mkdir "$git_dir" || die "cannot create git dir, \"$git_dir\""; }
test -d "$state_dir" \
  || { mkdir "$state_dir" || die "cannot create state dir, \"$state_dir\""; }
if test x"$user_map" = x; then
  test -f "$state_dir/user-map" \
    && user_map="$state_dir/user-map" \
    || user_map=/dev/null
else
  cat "$user_map" > $state_dir/user-map \
    || die "can't read user-map file: $user_map"
fi

user_map=$state_dir/user-map
cvs_repo=$state_dir/cvsrepo

# If this succeeds, then the key-file is valid
t="$rsync_url/$module/$key_file"
rsync "$t" > /dev/null \
  || die "invalid key-file? failed to rsync $t"

excl_opt=
for i in $(echo $exclude); do
    excl_opt="$excl_opt --exclude $module/$i/"
    excl_opt="$excl_opt --exclude '$module/$i/**'"
done

rsync -az $verbose		\
  --delete			\
  --delete-excluded		\
  $excl_opt			\
  --include "$module/"		\
  --include "$module/**"	\
  --exclude '*'			\
 $rsync_url/ $cvs_repo/$module

# Handle two different repository layouts:
# - savannah-style, where there's an extra layer of hierarchy, e.g.,
# gnulib/gnulib/ FIXME: finish this...
new_repo=$cvs_repo/$module
if test -d $new_repo && test -d $new_repo/$module; then
  cvs_repo=$new_repo
  rm -rf $cvs_repo/CVSROOT
fi
cvs -d $cvs_repo init

rsync_change_timestamp=$state_dir/rsync-change-timestamp

# See if "find" run exposes changes in name, mtime, or mode bits.
rsync_pulled_changes=yes
f=$state_dir/find
(cd $cvs_repo/$module && find . -type f -printf '%m %T@ %P\0') > $f.t
test -f $f && cmp $f $f.t > /dev/null \
  && rsync_pulled_changes=no

move_if_change $f.t $f

n_minutes=10

# If the previous "git cvsimport" run was too soon after the rsync that
# changed the CVS repository, then run it again, even if the latest
# rsync pulled in no changes.

need_git_cvsimport=yes
if test $rsync_pulled_changes = yes; then
  touch $rsync_change_timestamp
else
  if test -f $rsync_change_timestamp; then
    if timestamp_old_enough $rsync_change_timestamp $n_minutes; then
      # If there have been no changes for $n_minutes, remove the timestamp
      # file so that subsequent no-rsync-change iterations won't bother
      # to run "git cvsimport".  However, this one must still run it.
      rm $rsync_change_timestamp
    fi
  else
    need_git_cvsimport=no
  fi
fi

test $need_git_cvsimport = no && Exit 0

# Be sure to use cvsps-2.2b1, not 2.1
export PATH=/home/meyering/bin:$PATH

# Set HOME, so cvsps leaves its .cvspsrc file here, not in ~/.
export HOME=$state_dir
rm -rf $HOME/.cvsps
git cvsimport    \
  $verbose       \
  -A $user_map   \
  -d $cvs_repo   \
  -C $git_dir    \
  -p -z,120      \
  -o master      \
  -k -i          \
  $module

# Clean up (needed because git cvsimport's -i option doesn't work).
# Remove everything in $git_dir except .git/.
find $git_dir -mindepth 1 -maxdepth 1 -name '.git' -prune -o -print0 \
  | xargs -0 rm -rf

Exit 1

## Local Variables:
## eval: (add-hook 'write-file-hooks 'time-stamp)
## time-stamp-start: "VERSION='"
## time-stamp-format: "%:y-%02m-%02d %02H:%02M"
## time-stamp-time-zone: "UTC"
## time-stamp-end: "' # UTC"
## End:

The crontab entry runs this command: "mirror-sw gdb"