#!/usr/bin/perl -w # # mirror_list.pl -- generate various Debian mirror lists # Copyright (C) 1998 James Treacy # Copyright (C) 2000-2002, 2007-2008 Josip Rodin # Copyright (C) 2005 Joey Hess # Copyright (C) 2016, 2021 Peter Palfrader use strict; use English; use HTML::Entities; use List::Util qw(max); binmode(STDOUT => ':utf8'); require 5.001; my @filter_arches=qw(); # Architectures not to list. my $officialsiteregex = q{^ftp\d?(?:\.wa)?\...\.debian\.org$}; my $encode = '<>&"\''; use Getopt::Long; my ($mirror_source, $output_type, $help); my %opthash = ( "mirror|m=s" => \$mirror_source, "type|t=s" => \$output_type, "help|h!" => \$help, ); Getopt::Long::config('no_getopt_compat', 'no_auto_abbrev'); GetOptions(%opthash) or die "error parsing options"; $output_type = 'html' if (! defined $output_type); $mirror_source = 'Mirrors.masterlist' if (! defined $mirror_source); my $last_modify = gmtime((stat($mirror_source))[9]); my (@mirror, %countries, %countries_sponsors, %longest); my ($count, $cdimagecount); my (%code_of_country, %plain_name_of_country); my %includedsites; my $globalsite; sub cmpbydomain { my $sitea = $mirror[$a]{site}; my $siteb = $mirror[$b]{site}; my @a = map(scalar reverse, split(/\./, (scalar reverse $sitea))); my @b = map(scalar reverse, split(/\./, (scalar reverse $siteb))); while (1) { if ((scalar @a == 0) || (scalar @b == 0)) { return ((scalar @a) <=> (scalar @b)); } my $ca = shift @a; my $cb = shift @b; my $cmp = $ca cmp $cb; return $cmp if ($cmp != 0); } return 0; } sub process_line { my ($lno, $line) = @_; my $field = ''; if ($line =~ /^Site:\s*(.+)\s*$/i) { my $site = $1; $globalsite = $site; $count++; $mirror[$count-1]{site} = $site; $mirror[$count-1]{_lno} = $lno; return; } elsif ($line =~ /^Alias(?:es)?:\s*(.+)\s*$/is) { push @{ $mirror[$count-1]{aliases} }, $_ foreach (split("\n", $1)); } elsif ($line =~ /^(\w+)-architecture:\s*(.+)\s*$/i && length $2) { my $key = "$1"."-architecture"; my @arches=split(' ', $2); foreach my $f (@filter_arches) { @arches=grep { ! /^$f$/ } @arches; } if (! @arches) { # Mirror only carries filtered architectures. $mirror[$count-1]{filtered}=1; } foreach my $f (@filter_arches) { @arches=grep { ! /^\!$f$/ } @arches; } if (@arches) { $mirror[$count-1]{$key}=\@arches; } } elsif ($line=~ /^((Archive|CDimage|Jigdo|Old)-(\w*)):\s*(.*)\s*$/i) { my $type = lc $1; my $value = $4; $mirror[$count-1]{method}{$type} = $value; if (!defined($longest{$type}) || length($value)+1 > $longest{$type}) { $longest{$type} = length($value)+1; } } elsif ($line =~ /^Includes:\s*(.+)\s*$/i) { push @{ $mirror[$count-1]{includes} }, $_ foreach (split(" ", $1)); } elsif ($line =~ /^Sponsor:\s*(.+)\s*$/i) { push @{ $mirror[$count-1]{sponsor} }, $1; } elsif ($line =~ /^([\w-]+):\s*(.+)\s*$/s) { $field = lc $1; my $value = $2; # Ignore X-* fields if ($field !~ /^x-/) { $mirror[$count-1]{$field} = $value; } if ($field eq 'country') { if ($value =~ /^(\w\w) (.+)$/) { $code_of_country{$value} = $1; $plain_name_of_country{$value} = $2; } else { die "strangely formatted Country field value: $value"; } } } else { die "Error: incorrect line format\n\"$line\"\n"; } } sub aptlines { foreach my $country (sort keys %countries) { print "\n$country\n"; print "-" x length($country); # underline print "\n"; foreach my $id (@{ $countries{$country} }) { my $archcomm=""; if ($mirror[$id]{'Archive-architecture'}) { $archcomm=" # ".join(" ", sort @{$mirror[$id]{'Archive-architecture'}})."\n"; } if (defined $mirror[$id]{method}{'archive-http'}) { print "deb http://$mirror[$id]{site}$mirror[$id]{method}{'archive-http'} stable main contrib non-free non-free-firmware$archcomm\n"; } } print "\n"; } } sub secondary_mirrors { # TODO make the text version not have such long lines. my $format = shift; die "must get format for secondary_mirrors()" unless $format; my $html = 1 if ($format eq 'html'); my $text = 1 if ($format eq 'text'); my $wml = 1 if ($format eq 'wml'); my $formatstring; if ($html || $text) { print "
" if $html;
print "Debian is distributed (";
print $html ? "mirrored" : "mirrored";
print ") on hundreds of servers on the Internet.\n";
print < " if $html;
print <
END
} else {
print <
" if $html;
print "\n";
print <
" if $html;
print "\n";
print <
" if $html;
print "\n";
}
sub primary_mirrors {
my $format = shift;
die "must get format for primary_mirrors()" unless $format;
my $html = 1 if ($format eq 'html');
my $text = 1 if ($format eq 'text');
my $wml = 1 if ($format eq 'wml');
my %primaries = ();
if ($html) {
print <
\n" if ($html);
}
# meant to be output into a file which is then included into a .wml file
# and processed by WML
sub primary_mirror_sponsors {
print <
END
} elsif ($text) {
print <Country
Site
Architectures
$countryplain
$mirror[$id]{site}$mirror[$id]{method}{'archive-http'}
$arches
END
} elsif ($text) {
printf " %-14s %-20s %-14s %s\n", $countryplain, $mirror[$id]{site}, $mirror[$id]{method}{'archive-http'}, $arches;
} elsif ($wml) {
# this creates a hash of with keys like \$country
\$countrysite\$elems[0]
\$elems[1]
EOM
}
}
EOF
}
print "
END
foreach my $country (sort keys %countries) {
foreach my $id (@{ $countries{$country} }) {
next unless ($mirror[$id]{site} =~ /$officialsiteregex/);
my $countrycode = $code_of_country{$country};
print <<${countrycode}c>
$mirror[$id]{site}
END
if (exists $mirror[$id]{includes}) {
my $numsubsites = @{ $mirror[$id]{includes} };
my $snum = 0;
foreach my $subsite (@{ $mirror[$id]{includes} }) {
# XXX Note this is a little bit wrong; if there is more than one id
# for a subsite, it just takes the first one. This problem
# could occur if a subsite begins mirroring some other arch,
# like amd64.
my $subsite_id;
foreach my $mid (0..$#mirror) {
if ($mirror[$mid]{site} eq $subsite) {
$subsite_id=$mid;
last;
}
}
die "can't find $subsite, wtf?!" unless defined $subsite_id; # must be an error
die "can't find sponsor for $subsite, wtf?!" unless defined $mirror[$subsite_id]{sponsor}; # must be an error
my $numsponsors = @{ $mirror[$subsite_id]{sponsor} };
my $num = 0;
my ($sponsorname, $sponsorurl);
foreach my $sponsor (@{ $mirror[$subsite_id]{sponsor} }) {
if ($sponsor =~ /^(.+) (https?:.*)$/) {
$sponsorname = $1;
$sponsorurl = $2;
} else {
warn "can't find sponsor URL for sponsor $sponsor of $subsite";
$sponsorname = $sponsor;
}
encode_entities($sponsorname, $encode);
if ($sponsorurl) {
print "$sponsorname";
} else {
print "$sponsorname";
}
$num++;
print ", " unless ($num >= $numsponsors);
}
$snum++;
print ", " unless ($snum >= $numsubsites);
}
} else {
die "$mirror[$id]{site} has no sponsor\n" unless exists $mirror[$id]{sponsor}; # must be an error
my $numsponsors = @{ $mirror[$id]{sponsor} };
my $num = 0;
my ($sponsorname, $sponsorurl);
foreach my $sponsor (@{ $mirror[$id]{sponsor} }) {
if ($sponsor =~ /^(.+) (http:.*)$/) {
$sponsorname = $1;
$sponsorurl = $2;
} else {
warn "can't find sponsor URL for sponsor $sponsor of $mirror[$id]{site}";
$sponsorname = $sponsor;
}
encode_entities($sponsorname, $encode);
if ($sponsorurl) {
print "$sponsorname";
} else {
print "$sponsorname";
}
$num++;
print ", " unless ($num >= $numsponsors);
}
}
print <
END
foreach my $country (sort keys %countries) {
next unless $countries_sponsors{$country};
foreach my $id (@{ $countries_sponsors{$country} }) {
# sites which have Includes don't have to have Sponsor, the included ones
# have it; and those are looped over separately anyway, so no need to repeat
next if (exists $mirror[$id]{includes});
my $countrycode = $code_of_country{$country};
print <${countrycode} <${countrycode}c>
$mirror[$id]{site}
END
if (exists $mirror[$id]{'included-in'}) {
print "
(";
print join (", ", @{ $mirror[$id]{'included-in'} });
print ")";
}
print <
END
my $numsponsors = @{ $mirror[$id]{sponsor} };
my $num = 0;
my ($sponsorname, $sponsorurl);
foreach my $sponsor (@{ $mirror[$id]{sponsor} }) {
if ($sponsor =~ /^(.+) (https?:.*)$/) {
$sponsorname = $1;
$sponsorurl = $2;
} else {
warn "can't find sponsor URL for sponsor $sponsor of $mirror[$id]{site}";
$sponsorname = $sponsor;
}
encode_entities($sponsorname, $encode);
if ($sponsorurl) {
print "$sponsorname";
} else {
print "$sponsorname";
}
$num++;
print ",\n" unless ($num >= $numsponsors);
}
print "\n";
print < \n";
foreach $line (sort langcmp @cdmirrors)
{
print "
\n";