doc/guide: fill in the MASTER_SITE variables automatically

Keeping these two lists in sync is not something that humans should do.

5 files changed, 94 insertions, 73 deletions

diff --git a/doc/guide/Makefile b/doc/guide/Makefile
index b73ab86287e..73d0972b540 100644
--- a/doc/guide/Makefile
+++ b/doc/guide/Makefile
@@ -1,4 +1,4 @@
-# $NetBSD: Makefile,v 1.54 2019/04/29 16:18:41 rillig Exp $
+# $NetBSD: Makefile,v 1.55 2019/05/05 18:36:05 rillig Exp $
 
 DISTNAME=		pkgsrc-guide-${PKGVERSION}
 CATEGORIES=		# empty
@@ -66,7 +66,7 @@ pre-extract:
 pre-configure: generate-help-topics
 generate-help-topics: .PHONY
 	${RUN} ${MAKE} help topic=:index > ${WRKSRC}/help-topics.data
-	${RUN} cd ${WRKSRC} && python help-topics.gen.py
+	${RUN} cd ${WRKSRC} && env PKGSRCDIR=${PKGSRCDIR} python fill-placeholders.py *.xml
 
 do-build:
 .for _output_ in ${OUTPUTS}
diff --git a/doc/guide/files/build.xml b/doc/guide/files/build.xml
index 2fec969c79c..d24f4a5a03f 100644
--- a/doc/guide/files/build.xml
+++ b/doc/guide/files/build.xml
@@ -1,4 +1,4 @@
-<!-- $NetBSD: build.xml,v 1.78 2017/05/26 17:59:37 leot Exp $ -->
+<!-- $NetBSD: build.xml,v 1.79 2019/05/05 18:36:05 rillig Exp $ -->
 
 <chapter id="build">
   <title>The build process</title>
@@ -249,41 +249,7 @@ MASTER_SITES=   -http://www.example.com/archive/v1.0.0.tar.gz
       packages.  The names of the variables should speak for
       themselves.</para>
 
-      <!-- sort mk/fetch/sites.mk | sed -n 's/\(^MA[A-Z_]*\).*/    ${\1}/p' -->
-
-<programlisting>
-${MASTER_SITE_APACHE}
-${MASTER_SITE_BACKUP}
-${MASTER_SITE_CYGWIN}
-${MASTER_SITE_DEBIAN}
-${MASTER_SITE_FREEBSD}
-${MASTER_SITE_FREEBSD_LOCAL}
-${MASTER_SITE_GENTOO}
-${MASTER_SITE_GNOME}
-${MASTER_SITE_GNU}
-${MASTER_SITE_GNUSTEP}
-${MASTER_SITE_HASKELL_HACKAGE}
-${MASTER_SITE_IFARCHIVE}
-${MASTER_SITE_KDE}
-${MASTER_SITE_MOZILLA}
-${MASTER_SITE_MOZILLA_ALL}
-${MASTER_SITE_MOZILLA_ESR}
-${MASTER_SITE_MYSQL}
-${MASTER_SITE_NETLIB}
-${MASTER_SITE_OPENOFFICE}
-${MASTER_SITE_OSDN}
-${MASTER_SITE_PERL_CPAN}
-${MASTER_SITE_PGSQL}
-${MASTER_SITE_RUBYGEMS}
-${MASTER_SITE_R_CRAN}
-${MASTER_SITE_SOURCEFORGE}
-${MASTER_SITE_SUNSITE}
-${MASTER_SITE_SUSE}
-${MASTER_SITE_TEX_CTAN}
-${MASTER_SITE_XCONTRIB}
-${MASTER_SITE_XEMACS}
-${MASTER_SITE_XORG}
-</programlisting>
+@master_sites@
 
       <para>Some explanations for the less self-explaining ones:
       <varname>MASTER_SITE_BACKUP</varname> contains backup sites
diff --git a/doc/guide/files/fill-placeholders.py b/doc/guide/files/fill-placeholders.py
new file mode 100644
index 00000000000..b0fa60b94fb
--- /dev/null
+++ b/doc/guide/files/fill-placeholders.py
@@ -0,0 +1,88 @@
+#! python
+# $NetBSD: fill-placeholders.py,v 1.1 2019/05/05 18:36:05 rillig Exp $
+
+"""
+Fills in some sections of data that are determined directly from the
+pkgsrc code, such as variable names or help topics.
+"""
+
+import filecmp
+import os
+import re
+import sys
+from typing import List, Match
+from xml.sax.saxutils import escape as to_xml
+
+pkgsrcdir = os.environ['PKGSRCDIR']
+
+
+def read_lines(filename: str) -> List[str]:
+    with open(filename, 'r') as f:
+        return f.readlines()
+
+
+def help_topics() -> str:
+    # type="vert" would spread the columns over several pages,
+    # starting with aaaa | pppp on the first page, which is
+    # unintuitive in both the PDF and the HTML version.
+    out = ['<simplelist type="horiz" columns="2">']
+    for line in read_lines('help-topics.data')[2:]:
+        topic = line.strip()
+        out.append(f'<member>{to_xml(topic)}</member>')
+    out.append('</simplelist>')
+
+    return '\n'.join(out)
+
+
+def master_sites() -> str:
+    sites = []
+
+    for line in read_lines(f'{pkgsrcdir}/mk/fetch/sites.mk'):
+        m = re.match(r'^(MAS\w+)', line)
+        if m:
+            sites.append(m[1])
+
+    out = ['<simplelist type="horiz" columns="2">']
+    for site in sorted(sites):
+        out.append(f'<member>{to_xml(site)}</member>')
+    out.append('</simplelist>')
+
+    return '\n'.join(out)
+
+
+def process(filename: str, placeholders: set):
+    tmpl_filename = f'{filename}.tmpl'
+    tmp_filename = f'{filename}.tmp'
+    in_filename = tmpl_filename if os.path.isfile(tmpl_filename) else filename
+    out_filename = filename
+
+    phs = dict()
+    for p in placeholders:
+        phs[p.__name__] = p
+
+    def repl(m: Match):
+        return phs[m[1]]() if m[1] in phs else m[0]
+
+    out = []
+    for line in read_lines(in_filename):
+        out.append(re.sub(r'@(\w+)@', repl, line))
+
+    with open(tmp_filename, 'w') as f:
+        f.writelines(out)
+
+    if filecmp.cmp(tmp_filename, out_filename):
+        os.remove(tmp_filename)
+    elif in_filename == tmpl_filename:
+        os.replace(tmp_filename, out_filename)
+    else:
+        os.rename(out_filename, tmpl_filename)
+        os.rename(tmp_filename, out_filename)
+
+
+def main():
+    for filename in sys.argv[1:]:
+        process(filename, {help_topics, master_sites})
+
+
+if __name__ == '__main__':
+    main()
diff --git a/doc/guide/files/help-topics.gen.py b/doc/guide/files/help-topics.gen.py
deleted file mode 100644
index 78c0c816385..00000000000
--- a/doc/guide/files/help-topics.gen.py
+++ /dev/null
@@ -1,31 +0,0 @@
-#! python
-# $NetBSD: help-topics.gen.py,v 1.1 2019/04/29 16:18:41 rillig Exp $
-
-import os
-
-tmpl_file = "help-topics.tmpl.xml"
-data_file = "help-topics.data"
-out_file = "help-topics.xml"
-
-
-def merge():
-    def read_lines(filename):
-        with open(filename) as f:
-            return f.readlines()
-
-    out = []
-    for tmpl_line in read_lines(tmpl_file):
-        if '@topic@' in tmpl_line:
-            for topic in read_lines(data_file)[2:]:
-                xml_topic = topic.replace('&', '%amp;').replace('<', '&lt;')
-                out.append(tmpl_line.replace('@topic@', xml_topic))
-        else:
-            out.append(tmpl_line)
-
-    with open(f'{out_file}.tmp', 'w') as f:
-        f.writelines(out)
-    os.rename(f'{out_file}.tmp', out_file)
-
-
-if __name__ == '__main__':
-    merge()
diff --git a/doc/guide/files/help-topics.tmpl.xml b/doc/guide/files/help-topics.xml
index 5ed9bc11f1e..60a00770431 100644
--- a/doc/guide/files/help-topics.tmpl.xml
+++ b/doc/guide/files/help-topics.xml
@@ -1,4 +1,4 @@
-<!-- $NetBSD: help-topics.tmpl.xml,v 1.1 2019/04/28 15:22:24 rillig Exp $ -->
+<!-- $NetBSD: help-topics.xml,v 1.1 2019/05/05 18:36:05 rillig Exp $ -->
 
 <appendix id="help-topics">
 <title>Help topics</title>
@@ -8,8 +8,6 @@
 	when running <command>bmake help topic=:index</command>.
 </para>
 
-<itemizedlist>
-	<listitem><para>@topic@</para></listitem>
-</itemizedlist>
+@help_topics@
 
 </appendix>