[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]

Re: Using rsync to maintain local FC10 updates repo



Robert Moskowitz wrote:

So I wanted to do a simple rsync against one of the mirrors, and tried the script:

#!/bin/sh
rsync -auv rsync://mirrors.usc.edu/pub/linux/distributions/fedora/linux/updates/10/i386/ \
--delete --exclude=debug/ /repos/fedora/10/updates/i386

The rsync paths aren't like the ftp or http paths.  You want:

#!/bin/sh
rsync -auv rsync://mirrors.usc.edu/fedora/linux/updates/10/i386/ \
--delete --exclude=debug/ /repos/fedora/10/updates/i386

Except that this mirror's rsync seems to be broken!  ;)

Since Bruno included his rsync script, I thought I'd throw mine in for fun. You can create a configuration file like the attached fedora10.conf and then run "update_mirror fedora10.conf init" to build a new local mirror, and then run "update_mirror fedora10.conf update" in the future to pull in new updates. It's very space efficient, as it'll remove packages from the base when they appear in updates. The "result" directory under the MIRROR base will have a tree from which you can install.

I'm still using this to build CentOS mirrors myself, but I've given it up in favor of Revisor for Fedora mirrors, and will be dropping it soon for my CentOS mirrors. Revisor is a much better tool. It takes slightly longer to update than the rsync script, but it uses about the same amount of bandwidth, since it only downloads new packages, and it's more exact than the rsync script.
#!/bin/sh


MIRROR=/home/gordon/deploy/mirrors
SYNC="rsync -avH --delete --exclude debug --exclude '*-debuginfo-*.rpm'"

fail_cd () {
    echo "Could not cd to $1"
    exit 1
}

usage () {
	cat <<EOF
Use: update_mirror [config_file] [command]
Command must be one of "init", or "update".
EOF
	exit
}

init () {
	# Init will reinitialize all of the mirrors.
	for ARCH in $ARCHLIST ; do
		echo "Initializing ${NAME} release ${RELEASE} for ${ARCH}"
		BASEPATH="${MIRROR}/${NAME}-${RELEASE}-${ARCH}/${BASE}/"
		echo "Downloading base OS to ${BASEPATH}"
		mkdir -p ${MIRROR}/${NAME}-${RELEASE}-${ARCH}
		eval URL="\$URL_${BASE}"
		eval $SYNC ${URL} ${BASEPATH}
		# Build a list of packages in the base for update-only repos
		BASEPKGSYNCLIST=${BASEPATH%/}.synclist
		find ${BASEPATH} -type f -a -name '*.rpm' \
			| xargs rpm -qp --qf "+ %{NAME}-*.rpm\n" \
			> ${BASEPKGSYNCLIST}
		for REPO in $ADDITIONAL ; do
			REPOPATH="${MIRROR}/${NAME}-${RELEASE}-${ARCH}/${REPO}/"
			sync_additional 0
		done
		for REPO in $UPDATES ; do
			REPOPATH="${MIRROR}/${NAME}-${RELEASE}-${ARCH}/${REPO}/"
			sync_additional 1
		done
	done
}

sync_additional () {
	UPDATEONLY="$1"
	if [ "$UPDATEONLY" = "1" -a -r "${BASEPKGSYNCLIST}" ]; then
		SYNCLIST="--include-from ${BASEPKGSYNCLIST} --include '*/' --include 'repodata/**' --exclude '*'"
	fi
	echo "Updating \"${REPO}\" in ${REPOPATH}"
	eval URL="\$URL_${REPO}"
	eval $SYNC ${SYNCLIST} ${URL} ${REPOPATH}
	if test $? -ne 0 ; then
		echo "Update for \"${REPO}\" failed, aborting script"
		exit 0
	fi
	unset SYNCLIST
	# Set the marker if anything was actually downloaded.  This marker
	# and the beacon file were created in the update function.
	UPDATECOUNT=$(find "${REPOPATH}" -type f -cnewer "${BEACONPATH}" | wc -l)
	test $UPDATECOUNT -gt 0 && UPDATESFOUND=1
}

update () {
	for ARCH in $ARCHLIST ; do
		echo "Updating ${NAME} release ${RELEASE} for ${ARCH}"
		BASEPATH="${MIRROR}/${NAME}-${RELEASE}-${ARCH}/${BASE}/"
		BASEPKGSYNCLIST=${BASEPATH%/}.synclist
		RESULTPATH="${MIRROR}/${NAME}-${RELEASE}-${ARCH}/result/"
		BEACONPATH="$(mktemp /tmp/beacon.XXXXXX)"
		UPDATESFOUND=0
		for REPO in $ADDITIONAL ; do
			REPOPATH="${MIRROR}/${NAME}-${RELEASE}-${ARCH}/${REPO}/"
			sync_additional 0
		done
		for REPO in $UPDATES ; do
			REPOPATH="${MIRROR}/${NAME}-${RELEASE}-${ARCH}/${REPO}/"
			sync_additional 1
		done
		rm "${BEACONPATH}"
		if test "${UPDATESFOUND}" -eq 0 ; then
			echo "No new packages were found.  Skipping repomd update."
			continue
		fi
		echo "Cleaning up result directory"
		test -e ${RESULTPATH}/RPMS && rm -rf ${RESULTPATH}/RPMS
		mkdir -p ${RESULTPATH}/{repodata,images/xen,RPMS}
		ln -f ${BASEPATH}/repodata/*comps.xml ${RESULTPATH}/repodata/
		ln -f ${BASEPATH}/images/*.img ${RESULTPATH}/images/
		ln -f ${BASEPATH}/images/xen/* ${RESULTPATH}/images/xen/
		for REPO in $ADDITIONAL $UPDATES ; do
			REPOPATH="${MIRROR}/${NAME}-${RELEASE}-${ARCH}/${REPO}/"
			echo "Removing packages which exist in \"${REPO}\" from base"
			# If any package in this repository exists in the base, remove it.
			BASEPKGLIST=$(mktemp /tmp/baselist.XXXXXX)
			REPOPKGLIST=$(mktemp /tmp/repolist.XXXXXX)
			find ${BASEPATH} -type f -a -name '*.rpm' \
				| xargs rpm -qp --qf "%{NAME}:%{NAME}-%{VERSION}-%{RELEASE}.%{ARCH}.rpm\n" \
				> ${BASEPKGLIST}
			find ${REPOPATH} -type f -a -name '*.rpm' \
				| xargs rpm -qp --qf "%{NAME}\n" \
				| sort | uniq \
				> ${REPOPKGLIST}
			egrep "^("$(tr '\n' '|' < ${REPOPKGLIST})"):" ${BASEPKGLIST} \
				| cut -f2- -d: \
				| while read file ; do
					find ${BASEPATH} -type f -a -name "${file}"
				done \
				| xargs --no-run-if-empty rm
			rm -f ${BASEPKGLIST} ${REPOPKGLIST}
			find ${REPOPATH} -type f -a -name '*.rpm' \
				| xargs --no-run-if-empty ln -t ${RESULTPATH}/RPMS/
		done
		find ${BASEPATH} -type f -a -name '*.rpm' \
			| xargs --no-run-if-empty ln -t ${RESULTPATH}/RPMS/
		echo "Updating repomd data in ${RESULTPATH}"
		# Update the repodata that anaconda uses.
		pushd ${RESULTPATH}
			createrepo -g repodata/*comps.xml ${RESULTPATH}
		popd
	done
}

test -f "$1" || usage
. $1

case $2 in
	init)
		init
		;;
	update)
		update
		;;
	*)
		usage
		;;
esac

exit

NAME=fedora
RELEASE=10
ARCHLIST="i386 x86_64"
ARCHLIST="i386"
BASE="os"
UPDATES="updates"
URL_os='rsync://mirrors.kernel.org/fedora/releases/test/10-Preview/Fedora/${ARCH}/os/'
URL_updates='rsync://mirrors.kernel.org/fedora/development/${ARCH}/os/'
#URL_os='rsync://mirrors.kernel.org/fedora/releases/${RELEASE}/Fedora/${ARCH}/os/'
#URL_updates='rsync://mirrors.kernel.org/fedora/updates/${RELEASE}/${ARCH}/'
#URL_os='rsync://mirror.anl.gov/fedora/linux/releases/${RELEASE}/Fedora/${ARCH}/os/'
#URL_updates='rsync://mirror.anl.gov/fedora/linux/updates/${RELEASE}/${ARCH}/'

[Date Prev][Date Next]   [Thread Prev][Thread Next]   [Thread Index] [Date Index] [Author Index]