aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorMichele Locati <michele@locati.it>2018-03-15 10:13:51 +0100
committerMichele Locati <michele@locati.it>2018-03-15 11:32:45 +0100
commit2f5505bcbb4c32811a18863c6102bbb552dcc6c7 (patch)
treec297c281c95d6de6d20614207107e1d787c6ff36
parent6cb2a990ba107284a5eee9cd1b79d5b4de903fe0 (diff)
downloadincremental-git-filter-branch-2f5505bcbb4c32811a18863c6102bbb552dcc6c7.zip
incremental-git-filter-branch-2f5505bcbb4c32811a18863c6102bbb552dcc6c7.tar.gz
incremental-git-filter-branch-2f5505bcbb4c32811a18863c6102bbb552dcc6c7.tar.bz2
Tag remapping: avoid recursion problems, limit lookup depth; catch git errors
-rw-r--r--README.md2
-rwxr-xr-xbin/incremental-git-filterbranch.sh116
2 files changed, 76 insertions, 42 deletions
diff --git a/README.md b/README.md
index 66fe3eb..2da99a2 100644
--- a/README.md
+++ b/README.md
@@ -27,7 +27,7 @@ Get the script and read the syntax using the `--help` option.
```sh
./incremental-git-filterbranch.sh \
--whitelist 'develop master rx:release\/.*' \
- --tags all \
+ --tags all --tags-max-history-lookup 10 \
https://github.com/concrete5/concrete5.git \
'--prune-empty --subdirectory-filter concrete' \
git@github.com:concrete5/concrete5-core.git
diff --git a/bin/incremental-git-filterbranch.sh b/bin/incremental-git-filterbranch.sh
index 618f390..3a0e48e 100755
--- a/bin/incremental-git-filterbranch.sh
+++ b/bin/incremental-git-filterbranch.sh
@@ -32,6 +32,7 @@ usage () {
${0} [-h | --help] [--workdir <workdirpath>]
[--whitelist <whitelist>] [--blacklist <blacklist>]
[--tags (visited|all|none)]
+ [--tags-max-history-lookup <depth>]
[--no-hardlinks] [--no-atomic] [--no-lock] [--]
<sourcerepository> <filter> <destinationrepository>
Apply git filter-branch in an incremental way
@@ -50,6 +51,9 @@ Where:
visited: process only the tags visited (default)
none: do not process any tag
all: process all tags
+--tags-max-history-lookup
+ limit the depth when looking for best matched filtered commit when --tags is 'all'.
+ By default this value is 20.
--blacklist <blacklist>
a whitespace-separated list of branches to be excluded from the process.
Multiple options can be specified.
@@ -83,6 +87,7 @@ readParameters () {
ATOMIC='--atomic'
NO_LOCK=''
PROCESS_TAGS='visited'
+ PROCESS_TAGS_MAXHISTORYLOOKUP=20
while :
do
if test $# -lt 1
@@ -147,6 +152,22 @@ readParameters () {
esac
shift 2
;;
+ --tags-max-history-lookup)
+ if test $# -lt 2
+ then
+ usage 'Not enough arguments'
+ fi
+ PROCESS_TAGS_MAXHISTORYLOOKUP="${BRANCH_BLACKLIST} ${2}"
+ if ! test "${PROCESS_TAGS_MAXHISTORYLOOKUP}" -eq "${PROCESS_TAGS_MAXHISTORYLOOKUP}" 2>/dev/null
+ then
+ usage "Value of ${readParameters_currentArgument} should be numeric"
+ fi
+ if test "${PROCESS_TAGS_MAXHISTORYLOOKUP}" -lt 1
+ then
+ usage "Value of ${readParameters_currentArgument} should be greater than 0"
+ fi
+ shift 2
+ ;;
--no-hardlinks)
NO_HARDLINKS='--no-hardlinks'
shift 1
@@ -261,30 +282,13 @@ checkEnvironment () {
die 'The flock command is not available. You may want to use --no-lock option to avoid using it (but no concurrency check will be performed).'
fi
fi
- if ! command -v git >/dev/null
- then
- die 'The required git command is not available.'
- fi
- if ! command -v sed >/dev/null
- then
- die 'The required sed command is not available.'
- fi
- if ! command -v grep >/dev/null
- then
- die 'The required grep command is not available.'
- fi
- if ! command -v cut >/dev/null
- then
- die 'The required cut command is not available.'
- fi
- if ! command -v sort >/dev/null
- then
- die 'The required sort command is not available.'
- fi
- if ! command -v md5sum >/dev/null
- then
- die 'The required md5sum command is not available.'
- fi
+ for checkEnvironment_command in git sed grep md5sum
+ do
+ if ! command -v "${checkEnvironment_command}" >/dev/null
+ then
+ die "The required ${checkEnvironment_command} command is not available."
+ fi
+ done
checkEnvironment_vMin='2.15.0'
checkEnvironment_vCur=$(git --version | cut -d ' ' -f3)
checkEnvironment_vWork=$(printf '%s\n%s' "${checkEnvironment_vCur}" "${checkEnvironment_vMin}" | sort -t '.' -n -k1,1 -k2,2 -k3,3 -k4,4 | head -n 1)
@@ -351,11 +355,13 @@ getSourceRepositoryTagsInBranch () {
git -C "${SOURCE_REPOSITORY_DIR}" tag --list --merged "refs/heads/${1}" 2>/dev/null || true
}
+
getTagList () {
# List all tags and takes only the part after "refs/heads/"
git -C "${1}" show-ref --tags | sed -E 's:^.*?refs/tags/::' || true
}
+
branchInList () {
branchInList_branch="${1}"
branchInList_list="${2}"
@@ -383,6 +389,7 @@ branchInList () {
return 1
}
+
getBranchesToProcess () {
echo '# Determining branches to be processed'
WORK_BRANCHES=''
@@ -410,6 +417,7 @@ getBranchesToProcess () {
fi
}
+
prepareWorkerRepository () {
prepareWorkerRepository_haveToCreateRepo=1
if test -f "${WORKER_REPOSITORY_DIR}/.git/config"
@@ -441,6 +449,7 @@ prepareWorkerRepository () {
fi
}
+
processBranch () {
processBranch_branch="${1}"
echo ' - fetching'
@@ -478,9 +487,10 @@ processBranch () {
# shellcheck disable=SC2016
processBranch_tagNameFilter='read -r tag; printf "filter-branch/converted-tags/%s" "${tag}"'
fi
- # shellcheck disable=SC2086
rm -rf "${WORKER_REPOSITORY_DIR}.map"
- if git -C "${WORKER_REPOSITORY_DIR}" filter-branch \
+ exec 3>&1
+ # shellcheck disable=SC2086
+ if processBranch_stdErr=$(git -C "${WORKER_REPOSITORY_DIR}" filter-branch \
${FILTER} \
--remap-to-ancestor \
--tag-name-filter "${processBranch_tagNameFilter}" \
@@ -488,8 +498,22 @@ processBranch () {
--original "refs/filter-branch/originals/${processBranch_branch}" \
--state-branch "refs/filter-branch/state" \
--force \
- -- "${processBranch_range}"
+ -- "${processBranch_range}" \
+ 1>&3 2>&1 | tee /dev/stderr
+ )
+ then
+ processBranch_rc=0
+ else
+ processBranch_rc=1
+ fi
+ exec 3>&-
+ if test "${processBranch_rc}" -ne 0
then
+ if test -n "${processBranch_stdErr##*Found nothing to rewrite*}"
+ then
+ die 'git failed'
+ fi
+ else
if test "${PROCESS_TAGS}" = 'all' -a -n "${processBranch_tags}"
then
processBranchTag_availableTags="$(git -C "${WORKER_REPOSITORY_DIR}" tag --list | grep -E '^filter-branch/converted-tags/' | sed -E 's:^filter-branch/converted-tags/::')"
@@ -501,35 +525,38 @@ processBranch () {
fi
done
fi
- else
- # May fail with "Found nothing to rewrite"
- :
fi
echo " - storing state"
git -C "${WORKER_REPOSITORY_DIR}" branch -f "filter-branch/filtered/${processBranch_branch}" FETCH_HEAD
fi
}
+
getWorkingTagHash () {
- git -C "${WORKER_REPOSITORY_DIR}" tag -l --format='%(objectname)' -- "${1}"
+ git -C "${WORKER_REPOSITORY_DIR}" rev-list -n 1 "refs/tags/${1}" 2>/dev/null || true
}
+
getTranslatedNearestCommitHash () {
- getTranslatedNearestCommitHash_originalHash="${1}"
- getTranslatedNearestCommitHash_translatedHash="$(cat "${WORKER_REPOSITORY_DIR}.map" | grep -E "^${getTranslatedNearestCommitHash_originalHash}:" | sed -E 's_^.+?:__')"
- if test -n "${getTranslatedNearestCommitHash_translatedHash}"
+ # $1: the original hash
+ # $2 the allowed history depth
+ if test "${2}" -lt 1 -o -z "${PROCESS_TAGS_VISITEDHASHES##* ${1} *}"
+ then
+ return 1
+ fi
+ PROCESS_TAGS_VISITEDHASHES="${PROCESS_TAGS_VISITEDHASHES}${1} "
+ if getTranslatedNearestCommitHash_v="$(grep -E "^${1}:" "${WORKER_REPOSITORY_DIR}.map")"
then
- printf '%s' "${getTranslatedNearestCommitHash_translatedHash}"
+ printf '%s' "${getTranslatedNearestCommitHash_v#${1}:}"
return 0
fi
- getTranslatedNearestCommitHash_originalHashParents="$(git -C "${WORKER_REPOSITORY_DIR}" rev-list --parents -n 1 "${getTranslatedNearestCommitHash_originalHash}")"
- for getTranslatedNearestCommitHash_originalHashParent in ${getTranslatedNearestCommitHash_originalHashParents}
+ for getTranslatedNearestCommitHash_v in $(git -C "${WORKER_REPOSITORY_DIR}" rev-list --parents -n 1 "${1}")
do
- if test "${getTranslatedNearestCommitHash_originalHashParent}" != "${getTranslatedNearestCommitHash_originalHash}"
+ if test "${getTranslatedNearestCommitHash_v}" != "${1}"
then
- if getTranslatedNearestCommitHash_translatedHash="$(getTranslatedNearestCommitHash "${getTranslatedNearestCommitHash_originalHashParent}")"
+ if getTranslatedNearestCommitHash_v="$(getTranslatedNearestCommitHash "${getTranslatedNearestCommitHash_v}" "$(( $2 - 1))")"
then
- printf '%s' "${getTranslatedNearestCommitHash_translatedHash}"
+ printf '%s' "${getTranslatedNearestCommitHash_v}"
return 0
fi
fi
@@ -537,8 +564,10 @@ getTranslatedNearestCommitHash () {
return 1
}
+
processNotConvertedTag () {
processNotConvertedTag_tag="${1}"
+ printf ' - remapping tag %s\n' "${processNotConvertedTag_tag}"
processNotConvertedTag_tagOriginalHash="$(getWorkingTagHash "${processNotConvertedTag_tag}")"
if test -z "${processNotConvertedTag_tagOriginalHash}"
then
@@ -549,7 +578,8 @@ processNotConvertedTag () {
then
git -C "${WORKER_REPOSITORY_DIR}" show refs/filter-branch/state:filter.map >"${WORKER_REPOSITORY_DIR}.map"
fi
- if processNotConvertedTag_commitHash="$(getTranslatedNearestCommitHash "${processNotConvertedTag_tagOriginalHash}")"
+ PROCESS_TAGS_VISITEDHASHES=' '
+ if processNotConvertedTag_commitHash="$(getTranslatedNearestCommitHash "${processNotConvertedTag_tagOriginalHash}" "${PROCESS_TAGS_MAXHISTORYLOOKUP}")"
then
printf '%s -> filter-branch/converted-tags/%s (%s -> %s)\n' "${processNotConvertedTag_tag}" "${processNotConvertedTag_tag}" "${processNotConvertedTag_tagOriginalHash}" "${processNotConvertedTag_commitHash}"
git -C "${WORKER_REPOSITORY_DIR}" tag --force "filter-branch/converted-tags/${processNotConvertedTag_tag}" "${processNotConvertedTag_commitHash}"
@@ -558,6 +588,7 @@ processNotConvertedTag () {
fi
}
+
processBranches () {
processBranches_pushRefSpec=''
for processBranches_branch in ${WORK_BRANCHES}
@@ -582,10 +613,12 @@ processBranches () {
git -C "${WORKER_REPOSITORY_DIR}" push --quiet --force ${ATOMIC} destination ${processBranches_pushRefSpec}
}
+
md5 () {
printf '%s' "${1}" | md5sum | sed -E 's: .*$::'
}
+
itemInList () {
for itemInList_item in ${2}
do
@@ -597,6 +630,7 @@ itemInList () {
return 1
}
+
readParameters "$@"
normalizeParameters
checkEnvironment

© 2014-2020 Faster IT GmbH | imprint | privacy policy