aboutsummaryrefslogtreecommitdiffstats
path: root/bin/incremental-git-filterbranch
blob: 2a785cd492f7d85dd1b2784011543aab614e068b (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
#!/bin/sh
#
# Wrapper for git-filter-branch so that we can use it in an incremental
# way.
#
# Copyright (c) Michele Locati, 2018
#
# MIT license
# https://github.com/concrete5/incremental-filter-branch/blob/master/LICENSE
#

# Exit immediately if a pipeline, a list, or a compound command, exits with a non-zero status.
set -o errexit
# Treat unset variables and parameters other than the special parameters "@" and "*" as an error when performing parameter expansion.
set -o nounset
# Set the Internal Field Separator
IFS=' 	
'

# Exit with 1.
#
# Arguments:
#   $1: the message to be printed
die () {
	printf '%s\n' "${1}" >&2
	exit 1
}


# Print the usage and exit.
#
# Arguments:
#   $1 [optional]: if specified, a short usage message will be printed and we'll exit with 1;
#                  if not specified: a full syntax will be printed and we'll exit with 0
usage () {
	if test $# -eq 1
	then
		printf '%s\n\n%s\n' "${1}" "Type ${0} --help to get help" >&2
		exit 1
	fi
	printf '%s' "Usage:
${0} [-h | --help] [--workdir <workdirpath>]
	[--branch-whitelist <whitelist>] [--branch-blacklist <blacklist>]
	[--tag-whitelist <whitelist>] [--tag-blacklist <blacklist>]
	[--tags-plan (visited|all|none)]
	[--tags-max-history-lookup <depth>]
	[--prune-branches] [--prune-tags]
	[--no-hardlinks] [--no-atomic] [--no-lock] [--]
	<sourcerepository> <filter> <destinationrepository>
	Apply git filter-branch in an incremental way

Where:

--workdir workdirpath
	set the path to the directory where the temporary local repositories are created.
	By default, we'll use a directory named temp in the current directory.
--branch-whitelist <whitelist>
	a whitespace-separated list of branches to be included in the process.
	Multiple options can be specified.
	By default, all branches will be processed.
--branch-blacklist <blacklist>
	a whitespace-separated list of branches to be excluded from the process.
	Multiple options can be specified.
	By default, all branches will be processed.
	Blacklisted branches take the precedence over whitelisted ones.
--tag-whitelist <whitelist>
	a whitespace-separated list of tags to be included in the process.
	Multiple options can be specified.
--tag-blacklist <blacklist>
	a whitespace-separated list of tags to be excluded from the process.
	Multiple options can be specified.
	Blacklisted tags take the precedence over whitelisted ones.
--tags-plan
	how tags should be processed. This can be one of these values:
	- visited: process only the tags visited (default)
	- none: do not process any tag
	- all: process all tags
--tags-max-history-lookup
	limit the depth when looking for best matched filtered commit when --tags-plan is 'all'.
	By default this value is 50.
--prune-branches
	delete branches in the destination repository that do not exist anymore in the source repository,
	or that do not satisfy the whitelist/blacklist
--prune-tags
	delete tags in the destination repository that do not exist anymore in the source repository,
	or that do not satisfy the whitelist/blacklist
--no-hardlinks
	Do not create hard links (useful for file systems that don't support it).
--no-atomic
	Do not use an atomic transaction when pushing to the destination repository.
--no-lock
	Do not acquire an exclusive lock (useful for systems that don't have flock(1)).
sourcerepository
	The URL or path to the source repository.
filter
	The list of parameters to be passed to the git filter-branch command.
destinationrepository
	The URL or path to the destination repository.

You can prefix branch/tag names in both whitelists and blacklists with 'rx:': in this case a regular expression check will be performed.
For instance: --branch-whitelist 'master rx:release\/[0-9]+(\.[0-9]+)*' will match 'master' and 'release/1.1'.
Please remark that these regular expressions should comply the POSIX ERE (Extended Regular Expressions) syntax.
"
	exit 0
}


# Parse the command arguments and exits in case of errors.
#
# Arguments:
#   $@: all the command line parameters
readParameters () {
	WORK_DIRECTORY="$(pwd)/temp"
	BRANCH_WHITELIST=''
	BRANCH_BLACKLIST=''
	TAG_WHITELIST=''
	TAG_BLACKLIST=''
	TAGS_PLAN='visited'
	PROCESS_TAGS_MAXHISTORYLOOKUP=50
	NO_HARDLINKS=''
	ATOMIC='--atomic'
	NO_LOCK=''
	PRUNE_BRANCHES=0
	PRUNE_TAGS=0
	while :
	do
		if test $# -lt 1
		then
			usage 'Not enough arguments'
		fi
		readParameters_currentArgument="${1}"
		case "${readParameters_currentArgument}" in
			--)
				shift 1
				break
				;;
			-h|--help)
				usage
				;;
			--workdir)
				if test $# -lt 2
				then
					usage 'Not enough arguments'
				fi
				WORK_DIRECTORY="${2}"
				if test -z "${WORK_DIRECTORY}"
				then
					die 'The working directory option is empty'
				fi
				shift 2
				;;
			--branch-whitelist)
				if test $# -lt 2
				then
					usage 'Not enough arguments'
				fi
				BRANCH_WHITELIST="${BRANCH_WHITELIST} ${2}"
				shift 2
				;;
			--branch-blacklist)
				if test $# -lt 2
				then
					usage 'Not enough arguments'
				fi
				BRANCH_BLACKLIST="${BRANCH_BLACKLIST} ${2}"
				shift 2
				;;
			--tag-whitelist)
				if test $# -lt 2
				then
					usage 'Not enough arguments'
				fi
				TAG_WHITELIST="${TAG_WHITELIST} ${2}"
				shift 2
				;;
			--tag-blacklist)
				if test $# -lt 2
				then
					usage 'Not enough arguments'
				fi
				TAG_BLACKLIST="${TAG_BLACKLIST} ${2}"
				shift 2
				;;
			--tags-plan)
				if test $# -lt 2
				then
					usage 'Not enough arguments'
				fi
				case "${2}" in
					'all')
						TAGS_PLAN='all'
						;;
					'visited')
						TAGS_PLAN='visited'
						;;
					'none')
						TAGS_PLAN=''
						;;
					*)
						usage "Invalid value of the ${readParameters_currentArgument} option"
						;;
				esac
				shift 2
				;;
			--tags-max-history-lookup)
				if test $# -lt 2
				then
					usage 'Not enough arguments'
				fi
				PROCESS_TAGS_MAXHISTORYLOOKUP="${BRANCH_BLACKLIST} ${2}"
				if ! test "${PROCESS_TAGS_MAXHISTORYLOOKUP}" -eq "${PROCESS_TAGS_MAXHISTORYLOOKUP}" 2>/dev/null
				then
					usage "Value of ${readParameters_currentArgument} should be numeric"
				fi
				if test "${PROCESS_TAGS_MAXHISTORYLOOKUP}" -lt 1
				then
					usage "Value of ${readParameters_currentArgument} should be greater than 0"
				fi
				shift 2
				;;
			--prune-branches)
				PRUNE_BRANCHES=1
				shift 1
				;;
			--prune-tags)
				PRUNE_TAGS=1
				shift 1
				;;
			--no-hardlinks)
				NO_HARDLINKS='--no-hardlinks'
				shift 1
				;;
			--no-atomic)
				ATOMIC='--no-atomic'
				shift 1
				;;
			--no-lock)
				NO_LOCK='yes'
				shift 1
				;;
			-*)
				usage "Unknown option: ${readParameters_currentArgument}"
				;;
			*)
				break
				;;
		esac
	done
	if test -z "${TAGS_PLAN}"
	then
		if test -n "${TAG_WHITELIST}" -o -n "${TAG_BLACKLIST}"
		then
			die "You can't use --tag-whitelist or --tag-blacklist when you specify '--tags-plan none'"
		fi
		if test "${PRUNE_BRANCHES}" -ne 0 -o "${PRUNE_TAGS}" -ne 0
		then
			die "You can't use --prune-branches or --prune-tags when you specify '--tags-plan none'"
		fi
	fi
	if test $# -lt 3
	then
		usage 'Not enough arguments'
	fi
	if test $# -gt 3
	then
		usage 'Too many arguments'
	fi
	SOURCE_REPOSITORY_URL="${1}"
	if test -z "${SOURCE_REPOSITORY_URL}"
	then
		die 'The source repository location is empty.'
	fi
	SOURCE_REPOSITORY_URL=$(absolutizePath "${SOURCE_REPOSITORY_URL}")
	FILTER="${2}"
	if test -z "${FILTER}"
	then
		die 'The filter is empty.'
	fi
	# shellcheck disable=SC2086
	checkFilter ${FILTER}
	DESTINATION_REPOSITORY_URL="${3}"
	if test -z "${DESTINATION_REPOSITORY_URL}"
	then
		die 'The destination repository location is empty.'
	fi
	DESTINATION_REPOSITORY_URL=$(absolutizePath "${DESTINATION_REPOSITORY_URL}")
}


# Check if a string is a directory. If so, return its absolute path, otherwise the string itself.
#
# Arguments:
#   $1: the string to be checked
#
# Output:
#   The absolute path (if found), or $1
absolutizePath () {
	if test -d "${1}"
	then
		printf '%s' "$(cd "${1}" && pwd)"
	else
		printf '%s' "${1}"
	fi
}


# Check the contents of the <filter> argument and die in case of errors.
#
# Arguments:
#   $@: all parts of the filter
checkFilter () {
	checkFilter_some=0
	while test $# -ge 1
	do
		checkFilter_some=1
		checkFilter_optName="${1}"
		shift 1
		case "${checkFilter_optName}" in
			--setup)
				if test $# -lt 1
				then
					die "Invalid syntax in filter (${checkFilter_optName} without command)"
				fi
				shift 1
				;;
			--tag-name-filter)
				die "You can't use --tag-name-filter (it's handled automatically)"
				;;
			--*-filter)
				if test $# -lt 1
				then
					die "Invalid syntax in filter (${checkFilter_optName} without command)"
				fi
				shift 1
				;;
			--prune-empty)
				;;
			*)
				die "Invalid syntax in filter (unknown option: ${checkFilter_optName})"
				;;
		esac
	done
	if test ${checkFilter_some} -lt 1
	then
		die 'The filter is empty.'
	fi
}


# Check that the system has the required commands, and exit in case of problems.
checkEnvironment () {
	if test -z "${NO_LOCK}"
	then
		if ! command -v flock >/dev/null
		then
			die 'The flock command is not available. You may want to use --no-lock option to avoid using it (but no concurrency check will be performed).'
		fi
	fi
	for checkEnvironment_command in git sed grep
	do
		if ! command -v "${checkEnvironment_command}" >/dev/null
		then
			die "The required ${checkEnvironment_command} command is not available."
		fi
	done
	if ! $(echo '' | sed -E '' 2>/dev/null)
	then
		die 'The sed command does not support extended regular expressions.'
	fi
	if ! $(echo '' | grep -E '' 2>/dev/null)
	then
		die 'The grep command does not support extended regular expressions.'
	fi
	if command -v md5sum >/dev/null
	then
		MD5_COMMAND=md5sum
	elif command -v md5 >/dev/null
	then
		MD5_COMMAND=md5
	else
		die 'The required md5sum (or md5) command is not available.'
	fi
	checkEnvironment_vMin='2.16.0'
	checkEnvironment_vCur=$(git --version | cut -d ' ' -f3)
	checkEnvironment_vWork=$(printf '%s\n%s' "${checkEnvironment_vCur}" "${checkEnvironment_vMin}" | sort -t '.' -n -k1,1 -k2,2 -k3,3 -k4,4 | head -n 1)
	if test "${checkEnvironment_vWork}" != "${checkEnvironment_vMin}"
	then
		die "This script requires git ${checkEnvironment_vMin} (you have git ${checkEnvironment_vWork})."
	fi
}


# Initialize the working directory and associated variables, and exit in case of problems.
initializeEnvironment () {
	if ! test -d "${WORK_DIRECTORY}"
	then
		mkdir -p -- "${WORK_DIRECTORY}" || die "Failed to create the working directory ${WORK_DIRECTORY}"
	fi
	WORK_DIRECTORY=$(absolutizePath "${WORK_DIRECTORY}")
	SOURCE_REPOSITORY_DIR=${WORK_DIRECTORY}/source-$(getMD5 "${SOURCE_REPOSITORY_URL}")
	WORKER_REPOSITORY_DIR=${WORK_DIRECTORY}/worker-$(getMD5 "${SOURCE_REPOSITORY_URL}${DESTINATION_REPOSITORY_URL}")
}


# Acquire a lock (if allowed by options): we'll return from this function once the lock has been acquired
acquireLock () {
	if test -z "${NO_LOCK}"
	then
		exec 9>"${WORKER_REPOSITORY_DIR}.lock"
		while :
		do
			if flock -w 3 9
			then
				break
			fi
			echo 'Lock detected... Waiting that it becomes available...'
		done
	fi
}


# Create or update the mirror of the source repository.
prepareLocalSourceRepository () {
	prepareLocalSourceRepository_haveToCreateMirror=1
	if test -f "${SOURCE_REPOSITORY_DIR}/config"
	then
		echo '# Updating source repository'
		if git -C "${SOURCE_REPOSITORY_DIR}" remote update --prune
		then
			prepareLocalSourceRepository_haveToCreateMirror=0
		fi
	fi
	if test ${prepareLocalSourceRepository_haveToCreateMirror} -eq 1
	then
		echo '# Cloning source repository'
		rm -rf "${SOURCE_REPOSITORY_DIR}"
		git clone --mirror "${SOURCE_REPOSITORY_URL}" "${SOURCE_REPOSITORY_DIR}"
	fi
}


# Store in the SOURCE_BRANCHES variable the list of the branches in the source repository.
# Exit if no branch can be found.
getSourceRepositoryBranches () {
	echo '# Listing source branches'
	# List all branches and takes only the part after "refs/heads/", and store them in the SOURCE_BRANCHES variable
	SOURCE_BRANCHES=$(git -C "${SOURCE_REPOSITORY_DIR}" show-ref --heads | sed -E 's:^.* refs/heads/::')
	if test -z "${SOURCE_BRANCHES}"
	then
		die 'Failed to retrieve branch list'
	fi
}


# Get the tags that exist in a specific branch of the source repository.
#
# Arguments:
#   $1: the branch name
#
# Output:
#   The list of tags (one per line)
getSourceRepositoryTagsInBranch () {
	git -C "${SOURCE_REPOSITORY_DIR}" tag --list --merged "refs/heads/${1}" 2>/dev/null || true
}


# Get the list of all the tags that exist in a repository.
#
# Arguments:
#   $1: the directory of the repository
#
# Output:
#   The list of tags (one per line)
getTagList () {
	# List all tags and takes only the part after "refs/heads/"
	git -C "${1}" show-ref --tags | sed -E 's:^.* refs/tags/::' || true
}


# Check if a string is in a white/black list.
#
# Arguments:
#   $1: the string to be checked
#   $2: the white/black list
#
# Return:
#   0 (true): if the string is in the list
#   1 (false): if the string is not in the list
stringInList () {
	for stringInList_listItem in ${2}
	do
		if test -n "${stringInList_listItem}"
		then
			case "${stringInList_listItem}" in
				rx:*)
					stringInList_substring=$(printf '%s' "${stringInList_listItem}" | cut -c4-)
					if printf '%s' "${1}" | grep -Eq "^${stringInList_substring}$"
					then
						return 0
					fi
					;;
				*)
					if test "${1}" = "${stringInList_listItem}"
					then
						return 0
					fi
					;;
			esac
		fi
	done
	return 1
}


# Check if a string satisfies whitelist and blacklist checks.
#
# Arguments:
#   $1: the string to be checked
#   $2: the whitelist
#   $3: the blacklist
#
# Return:
#   0 (true): if the string satisfies the criteria
#   1 (false): if the string does not satisfy the criteria
stringPassesLists () {
	if stringInList "${1}" "${3}"
	then
		return 1
	fi
	if test -z "${2}"
	then
		return 0
	fi
	if stringInList "${1}" "${2}"
	then
		return 0
	fi
	return 1
}


# Store in the WORK_BRANCHES variable the list of the branches to be processed (checking the white/black lists).
# Die if no branch can be found.
getBranchesToProcess () {
	echo '# Determining the branches to be processed'
	WORK_BRANCHES=''
	for getBranchesToProcess_branch in ${SOURCE_BRANCHES}
	do
		if stringPassesLists "${getBranchesToProcess_branch}" "${BRANCH_WHITELIST}" "${BRANCH_BLACKLIST}"
		then
			WORK_BRANCHES="${WORK_BRANCHES} ${getBranchesToProcess_branch}"
		fi
	done
	if test -z "${WORK_BRANCHES}"
	then
		die 'None of the source branches passes the whitelist/blacklist filter'
	fi
}


# Create the worker repository (if it does not already exist)
prepareWorkerRepository () {
	prepareWorkerRepository_haveToCreateRepo=1
	if test -f "${WORKER_REPOSITORY_DIR}/config"
	then
		echo '# Checking working repository'
		if git -C "${WORKER_REPOSITORY_DIR}" rev-parse --git-dir >/dev/null 2>/dev/null
		then
			prepareWorkerRepository_haveToCreateRepo=0
		fi
	fi
	if test ${prepareWorkerRepository_haveToCreateRepo} -eq 1
	then
		echo '# Creating working repository'
		rm -rf "${WORKER_REPOSITORY_DIR}"
		git init --bare "${WORKER_REPOSITORY_DIR}"
		echo '# Adding mirror source repository to working repository'
		if ! git -C "${WORKER_REPOSITORY_DIR}" remote add source "${SOURCE_REPOSITORY_DIR}"
		then
			rm -rf "${WORKER_REPOSITORY_DIR}"
			exit 1
		fi
		echo '# Adding destination repository to working repository'
		if ! git -C "${WORKER_REPOSITORY_DIR}" remote add destination "${DESTINATION_REPOSITORY_URL}"
		then
			rm -rf "${WORKER_REPOSITORY_DIR}"
			exit 1
		fi
		echo '# Fetching data from cloned destination repository'
		if ! git -C "${WORKER_REPOSITORY_DIR}" fetch --prune destination
		then
			rm -rf "${WORKER_REPOSITORY_DIR}"
			exit 1
		fi
	fi
	git -C "${WORKER_REPOSITORY_DIR}" symbolic-ref HEAD refs/none
}


# Process a branch, fetching it from the mirror of the source repository.
# Any relevat tag will also be processed.
#
# Arguments:
#   $1: the name of the branch to work with
processBranch () {
	echo '  - fetching'
	git -C "${WORKER_REPOSITORY_DIR}" fetch --quiet --tags source "${1}"
	echo '  - setting current branch'
	git -C "${WORKER_REPOSITORY_DIR}" update-ref "refs/heads/filter-branch/source/${1}" "refs/remotes/source/${1}"
	git -C "${WORKER_REPOSITORY_DIR}" symbolic-ref HEAD "refs/heads/filter-branch/source/${1}"
	echo '  - determining delta'
	processBranch_range="filter-branch/result/${1}"
	processBranch_last=$(git -C "${WORKER_REPOSITORY_DIR}" show-ref -s "refs/heads/filter-branch/filtered/${1}" || true)
	if test -n "${processBranch_last}"
	then
		processBranch_range="${processBranch_last}..${processBranch_range}"
	fi
	processBranch_fetchHead=$(git -C "${WORKER_REPOSITORY_DIR}" rev-parse FETCH_HEAD)
	if test "${processBranch_last}" = "${processBranch_fetchHead}"
	then
		echo '  - nothing new, skipping'
	else
		echo '  - initializing filter'
		rm -f "${WORKER_REPOSITORY_DIR}/refs/filter-branch/originals/${1}/refs/heads/filter-branch/result/${1}"
		git -C "${WORKER_REPOSITORY_DIR}" branch --force "filter-branch/result/${1}" FETCH_HEAD
		rm -rf "${WORKER_REPOSITORY_DIR}.filter-branch"
		echo "  - filtering commits"
		processBranch_tags=''
		if test -z "${TAGS_PLAN}"
		then
			processBranch_tags=''
		else
			processBranch_tags=$(getSourceRepositoryTagsInBranch "${1}")
		fi
		if test -z "${processBranch_tags}"
		then
			processBranch_tagNameFilter=''
		else
			# shellcheck disable=SC2016
			processBranch_tagNameFilter='read -r tag; printf "filter-branch/converted-tags/%s" "${tag}"'
		fi
		rm -rf "${WORKER_REPOSITORY_DIR}.map"
		exec 3>&1
		# shellcheck disable=SC2086
		if processBranch_stdErr=$(git -C "${WORKER_REPOSITORY_DIR}" filter-branch \
			${FILTER} \
			--remap-to-ancestor \
			--tag-name-filter "${processBranch_tagNameFilter}" \
			-d "${WORKER_REPOSITORY_DIR}.filter-branch" \
			--original "refs/filter-branch/originals/${1}" \
			--state-branch "refs/filter-branch/state" \
			--force \
			-- "${processBranch_range}" \
			1>&3 2>&1 | tee /dev/stderr
		)
		then
			processBranch_rc=0
		else
			processBranch_rc=1
		fi
		exec 3>&-
		if test "${processBranch_rc}" -ne 0
		then
			if test -n "${processBranch_stdErr##*Found nothing to rewrite*}"
			then
				die 'git failed'
			fi
		else
			if test "${TAGS_PLAN}" = 'all' -a -n "${processBranch_tags}"
			then
				if ! processBranchTag_availableTags="$(git -C "${WORKER_REPOSITORY_DIR}" tag --list | grep -E '^filter-branch/converted-tags/' | sed -E 's:^filter-branch/converted-tags/::')"
				then
					processBranchTag_availableTags=''
				fi
				for processBranch_tag in ${processBranch_tags}
				do
					if ! itemInList "${processBranch_tag}" "${processBranchTag_availableTags}"
					then
						if stringPassesLists "${processBranch_tag}" "${TAG_WHITELIST}" "${TAG_BLACKLIST}"
						then
							processNotConvertedTag "${processBranch_tag}"
						fi
					fi
				done
			fi
		fi
		echo "  - storing state"
		git -C "${WORKER_REPOSITORY_DIR}" branch -f "filter-branch/filtered/${1}" FETCH_HEAD
	fi
}


# Print the SHA-1 hash of the tag of the work repository (if found).
#
# Arguments:
#   $1: the name of the tag
#
# Output:
#   The SHA-1 of the tag, or nothing if the tag does not exist
getWorkingTagHash () {
	git -C "${WORKER_REPOSITORY_DIR}" rev-list -n 1 "refs/tags/${1}" 2>/dev/null || true
}


# Tries to create a new translated tag, associating it to the nearest translated commit
#
# Arguments:
#   $1: the name of the tag to be translated
processNotConvertedTag () {
	printf '  - remapping tag %s... ' "${1}"
	processNotConvertedTag_tagOriginalHash="$(getWorkingTagHash "${1}")"
	if test -z "${processNotConvertedTag_tagOriginalHash}"
	then
		die "$(printf 'failed to get hash of tag %s' "${1}")"
	fi
	if test ! -f "${WORKER_REPOSITORY_DIR}.map"
	then
		git -C "${WORKER_REPOSITORY_DIR}" show refs/filter-branch/state:filter.map >"${WORKER_REPOSITORY_DIR}.map"
	fi
	processNotConvertedTag_translatedCommit=''
	for processNotConvertedTag_commitHash in $(git -C "${WORKER_REPOSITORY_DIR}" rev-list --date-order --max-count="${PROCESS_TAGS_MAXHISTORYLOOKUP}" "${processNotConvertedTag_tagOriginalHash}")
	do
		if processNotConvertedTag_mappedCommit="$(grep -E "^${processNotConvertedTag_commitHash}:" "${WORKER_REPOSITORY_DIR}.map")"
		then
			processNotConvertedTag_translatedCommit="${processNotConvertedTag_mappedCommit#${processNotConvertedTag_commitHash}:}"
			break
		fi
	done
	if test -z "${processNotConvertedTag_translatedCommit}"
	then
		printf 'nearest commit not found\n' >&2
	else
		printf 'mapping to commit %s\n' "${processNotConvertedTag_translatedCommit}"
		git -C "${WORKER_REPOSITORY_DIR}" tag --force "filter-branch/converted-tags/${1}" "${processNotConvertedTag_translatedCommit}"
	fi
}


# Remove already translated tags that do not exist in source repository anymore
removeTranslatedTags () {
	if test ${PRUNE_TAGS} -eq 0
	then
		# Superfluous
		return 0
	fi
	echo '# Listing currently converted tags'
	removeTranslatedTags_workerTags="$(git -C "${WORKER_REPOSITORY_DIR}" tag -l || true)"
	if test -z "${removeTranslatedTags_workerTags}"
	then
		return 0
	fi
	echo '# Listing tags in source repository'
	removeTranslatedTags_sourceTags="$(git -C "${WORKER_REPOSITORY_DIR}" ls-remote --quiet --tags source |  grep -Ev '\^\{\}$' | sed -E 's:^.*[ \t]refs/tags/::g')"
	echo '# Deleting previously converted tags no more existing in source repository'
	for removeTranslatedTags_workerTag in ${removeTranslatedTags_workerTags}
	do
		removeTranslatedTags_workerTagName="$(printf '%s' "${removeTranslatedTags_workerTag}" | sed -E 's:^filter-branch/converted-tags/::')"
		removeTranslatedTags_deleteTag=1
		if stringPassesLists "${removeTranslatedTags_workerTagName}" "${TAG_WHITELIST}" "${TAG_BLACKLIST}"
		then
			if test -n "${removeTranslatedTags_sourceTags}"
			then
				if itemInList "${removeTranslatedTags_workerTagName}" "${removeTranslatedTags_sourceTags}"
				then
					removeTranslatedTags_deleteTag=0
				fi
			fi
		fi
		if test ${removeTranslatedTags_deleteTag} -eq 1
		then
			printf '  - deleting translated tag %s\n' "${removeTranslatedTags_workerTag}"
			git -C "${WORKER_REPOSITORY_DIR}" tag -d "${removeTranslatedTags_workerTag}"
		fi
	done
}


# Process all the branches listed in the WORK_BRANCHES variable, and push the result to the destination repository.
processBranches () {
	processBranches_pushRefSpec=''
	for processBranches_branch in ${WORK_BRANCHES}
	do
		echo "# Processing branch ${processBranches_branch}"
		processBranch "${processBranches_branch}"
		processBranches_pushRefSpec="${processBranches_pushRefSpec} filter-branch/result/${processBranches_branch}:${processBranches_branch}"
	done
	if test -n "${TAGS_PLAN}"
	then
		echo '# Listing source tags'
		processBranches_sourceTags=$(getTagList "${SOURCE_REPOSITORY_DIR}")
		echo '# Determining destination tags'
		for processBranches_sourceTag in ${processBranches_sourceTags}
		do
			if stringPassesLists "${processBranches_sourceTag}" "${TAG_WHITELIST}" "${TAG_BLACKLIST}"
			then
				processBranches_rewrittenTag="filter-branch/converted-tags/${processBranches_sourceTag}"
				if git -C "${WORKER_REPOSITORY_DIR}" rev-list --max-count=0 "${processBranches_rewrittenTag}" 2>/dev/null
				then
					processBranches_pushRefSpec="${processBranches_pushRefSpec} ${processBranches_rewrittenTag}:${processBranches_sourceTag}"
				fi
			fi
		done
	fi
	echo "# Pushing to destination repository"
	# shellcheck disable=SC2086
	git -C "${WORKER_REPOSITORY_DIR}" push --quiet --force ${ATOMIC} destination ${processBranches_pushRefSpec}
}


# Get the tags to be removed from the destination repository
#
# Output:
#   - Empty string if no tag should be removed
#   - Space-separated list (including a final extra space) of full tag paths (refs/tags/...) otherwise
getPruneTags () {
	getPruneTags_remoteTags="$(git -C "${WORKER_REPOSITORY_DIR}" ls-remote --quiet --tags destination |  grep -Ev '\^\{\}$' | sed -E 's:^.*[ \t]refs/tags/::g')"
	if test -z "${getPruneTags_remoteTags}"
	then
		return 0
	fi
	getPruneTags_localTags=$(getTagList "${WORKER_REPOSITORY_DIR}")
	for getPruneTags_remoteTag in ${getPruneTags_remoteTags}
	do
		getPruneTags_doDelete=1
		if test -n "${getPruneTags_localTags}"
		then
			if itemInList "filter-branch/converted-tags/${getPruneTags_remoteTag}" "${getPruneTags_localTags}"
			then
				getPruneTags_doDelete=0
			fi
		fi
		if test ${getPruneTags_doDelete} -eq 1
		then
			printf 'refs/tags/%s ' "${getPruneTags_remoteTag}"
		fi
	done
}


# Delete in destination repository the non converted branches
getPruneBranches () {
	getPruneBranches_currentBranch=''
	getPruneBranches_remoteBranches="$(git -C "${WORKER_REPOSITORY_DIR}" ls-remote --quiet --heads destination | sed -E 's:^.*[ \t]refs/heads/::g')"
	for getPruneBranches_remoteBranch in ${getPruneBranches_remoteBranches}
	do
		if ! itemInList "${getPruneBranches_remoteBranch}" "${WORK_BRANCHES}"
		then
			if test -z "${getPruneBranches_currentBranch}"
			then
				getPruneBranches_currentBranch="$(git -C "${WORKER_REPOSITORY_DIR}" ls-remote --symref destination HEAD | head -1 | sed -E 's_^ref: +refs/heads/__' | cut -f 1 | cut -d ' ' -f 1)"
			fi
			if test "${getPruneBranches_currentBranch}" = "${getPruneBranches_remoteBranch}"
			then
				printf 'Remote branch %s will NOT be deleted since it is the current one\n' "${getPruneBranches_remoteBranch}" >&2
			else
				printf 'refs/heads/%s ' "${getPruneBranches_remoteBranch}"
			fi
		fi
	done
}


# Delete in destination repository the non converted branches and tags
pruneDestination () {
	pruneDestination_allRefs=''
	if test ${PRUNE_TAGS} -ne 0
	then
		echo '# Determining destination tags to be removed'
		pruneDestination_theseRefs="$(getPruneTags)"
		if test -n "${pruneDestination_theseRefs}"
		then
			pruneDestination_allRefs="${pruneDestination_allRefs}${pruneDestination_theseRefs}"
		fi
	fi
	if test ${PRUNE_BRANCHES} -ne 0
	then
		echo '# Determining destination branches to be removed'
		pruneDestination_theseRefs="$(getPruneBranches)"
		if test -n "${pruneDestination_theseRefs}"
		then
			pruneDestination_allRefs="${pruneDestination_allRefs}${pruneDestination_theseRefs}"
		fi
	fi
	if test -z "${pruneDestination_allRefs}"
	then
		return 0
	fi
	printf '# Deleting refs in destination repository (%s)\n' "${pruneDestination_allRefs% }"
	# shellcheck disable=SC2086
	git -C "${WORKER_REPOSITORY_DIR}" push --quiet --delete destination ${pruneDestination_allRefs% }
}


# Calculate the MD5 hash of a string.
#
# Arguments:
#   $1: the string for which you want the MD5 hash
#
# Output:
#   The MD5-1 hash of $1
getMD5 () {
	printf '%s' "${1}" | "${MD5_COMMAND}" | sed -E 's: .*$::'
}


# Check if a string is in a list (separated by spaces, tabs or new lines)
#
# Arguments:
#   $1: the string
#   $2: the list of strings
#
# Return:
#   0 (true): $1 is in $2
#   1 (false): $1 is not in $2
itemInList () {
	for itemInList_item in ${2}
	do
		if test "${1}" = "${itemInList_item}"
		then
			return 0
		fi
	done
	return 1
}


readParameters "$@"
checkEnvironment
initializeEnvironment
acquireLock
prepareLocalSourceRepository
getSourceRepositoryBranches
getBranchesToProcess
prepareWorkerRepository
removeTranslatedTags
processBranches
pruneDestination
echo "All done."

© 2014-2020 Faster IT GmbH | imprint | privacy policy