#!/usr/bin/env python3 # This script takes a manpage written in markdown and turns it into an html web # page and a nroff man page. The input file must have the name of the program # and the section in this format: NAME.NUM.md. The output files are written # into the current directory named NAME.NUM.html and NAME.NUM. The input # format has one extra extension: if a numbered list starts at 0, it is turned # into a description list. The dl's dt tag is taken from the contents of the # first tag inside the li, which is usually a p, code, or strong tag. The # cmarkgfm or commonmark lib is used to transforms the input file into html. # The html.parser is used as a state machine that both tweaks the html and # outputs the nroff data based on the html tags. # # Copyright (C) 2020 Wayne Davison # # This program is freely redistributable. import sys, os, re, argparse, subprocess, time from html.parser import HTMLParser CONSUMES_TXT = set('h1 h2 p li pre'.split()) HTML_START = """\ %s """ HTML_END = """\

%s

""" MAN_START = r""" .TH "%s" "%s" "%s" "%s" "User Commands" """.lstrip() MAN_END = """\ """ NORM_FONT = ('\1', r"\fP") BOLD_FONT = ('\2', r"\fB") UNDR_FONT = ('\3', r"\fI") NBR_DASH = ('\4', r"\-") NBR_SPACE = ('\xa0', r"\ ") md_parser = None def main(): fi = re.match(r'^(?P(?P.+/)?(?P(?P[^/]+)\.(?P\d+))\.md)$', args.mdfile) if not fi: die('Failed to parse NAME.NUM.md out of input file:', args.mdfile) fi = argparse.Namespace(**fi.groupdict()) if not fi.srcdir: fi.srcdir = './' fi.title = fi.prog + '(' + fi.sect + ') man page' fi.mtime = 0 git_dir = fi.srcdir + '.git' if os.path.lexists(git_dir): fi.mtime = int(subprocess.check_output(['git', '--git-dir', git_dir, 'log', '-1', '--format=%at'])) env_subs = { 'prefix': os.environ.get('RSYNC_OVERRIDE_PREFIX', None) } if args.test: env_subs['VERSION'] = '1.0.0' env_subs['libdir'] = '/usr' else: for fn in (fi.srcdir + 'version.h', 'Makefile'): try: st = os.lstat(fn) except: die('Failed to find', fi.srcdir + fn) if not fi.mtime: fi.mtime = st.st_mtime with open(fi.srcdir + 'version.h', 'r', encoding='utf-8') as fh: txt = fh.read() m = re.search(r'"(.+?)"', txt) env_subs['VERSION'] = m.group(1) with open('Makefile', 'r', encoding='utf-8') as fh: for line in fh: m = re.match(r'^(\w+)=(.+)', line) if not m: continue var, val = (m.group(1), m.group(2)) if var == 'prefix' and env_subs[var] is not None: continue while re.search(r'\$\{', val): val = re.sub(r'\$\{(\w+)\}', lambda m: env_subs[m.group(1)], val) env_subs[var] = val if var == 'srcdir': break with open(fi.fn, 'r', encoding='utf-8') as fh: txt = fh.read() txt = re.sub(r'@VERSION@', env_subs['VERSION'], txt) txt = re.sub(r'@LIBDIR@', env_subs['libdir'], txt) fi.html_in = md_parser(txt) txt = None fi.date = time.strftime('%d %b %Y', time.localtime(fi.mtime)) fi.man_headings = (fi.prog, fi.sect, fi.date, fi.prog + ' ' + env_subs['VERSION']) HtmlToManPage(fi) if args.test: print("The test was successful.") return for fn, txt in ((fi.name + '.html', fi.html_out), (fi.name, fi.man_out)): print("Wrote:", fn) with open(fn, 'w', encoding='utf-8') as fh: fh.write(txt) def html_via_commonmark(txt): return commonmark.HtmlRenderer().render(commonmark.Parser().parse(txt)) class HtmlToManPage(HTMLParser): def __init__(self, fi): HTMLParser.__init__(self, convert_charrefs=True) st = self.state = argparse.Namespace( list_state = [ ], p_macro = ".P\n", at_first_tag_in_li = False, at_first_tag_in_dd = False, dt_from = None, in_pre = False, in_code = False, html_out = [ HTML_START % fi.title ], man_out = [ MAN_START % fi.man_headings ], txt = '', ) self.feed(fi.html_in) fi.html_in = None st.html_out.append(HTML_END % fi.date) st.man_out.append(MAN_END) fi.html_out = ''.join(st.html_out) st.html_out = None fi.man_out = ''.join(st.man_out) st.man_out = None def handle_starttag(self, tag, attrs_list): st = self.state if args.debug: self.output_debug('START', (tag, attrs_list)) if st.at_first_tag_in_li: if st.list_state[-1] == 'dl': st.dt_from = tag if tag == 'p': tag = 'dt' else: st.html_out.append('
') elif tag == 'p': st.at_first_tag_in_dd = True # Kluge to suppress a .P at the start of an li. st.at_first_tag_in_li = False if tag == 'p': if not st.at_first_tag_in_dd: st.man_out.append(st.p_macro) elif tag == 'li': st.at_first_tag_in_li = True lstate = st.list_state[-1] if lstate == 'dl': return if lstate == 'o': st.man_out.append(".IP o\n") else: st.man_out.append(".IP " + str(lstate) + ".\n") st.list_state[-1] += 1 elif tag == 'blockquote': st.man_out.append(".RS 4\n") elif tag == 'pre': st.in_pre = True st.man_out.append(st.p_macro + ".nf\n") elif tag == 'code' and not st.in_pre: st.in_code = True st.txt += BOLD_FONT[0] elif tag == 'strong' or tag == 'b': st.txt += BOLD_FONT[0] elif tag == 'em' or tag == 'i': tag = 'u' # Change it into underline to be more like the man page st.txt += UNDR_FONT[0] elif tag == 'ol': start = 1 for var, val in attrs_list: if var == 'start': start = int(val) # We only support integers. break if st.list_state: st.man_out.append(".RS\n") if start == 0: tag = 'dl' attrs_list = [ ] st.list_state.append('dl') else: st.list_state.append(start) st.man_out.append(st.p_macro) st.p_macro = ".IP\n" elif tag == 'ul': st.man_out.append(st.p_macro) if st.list_state: st.man_out.append(".RS\n") st.p_macro = ".IP\n" st.list_state.append('o') st.html_out.append('<' + tag + ''.join(' ' + var + '="' + htmlify(val) + '"' for var, val in attrs_list) + '>') st.at_first_tag_in_dd = False def handle_endtag(self, tag): st = self.state if args.debug: self.output_debug('END', (tag,)) if tag in CONSUMES_TXT or st.dt_from == tag: txt = st.txt.strip() st.txt = '' else: txt = None add_to_txt = None if tag == 'h1': st.man_out.append(st.p_macro + '.SH "' + manify(txt) + '"\n') elif tag == 'h2': st.man_out.append(st.p_macro + '.SS "' + manify(txt) + '"\n') elif tag == 'p': if st.dt_from == 'p': tag = 'dt' st.man_out.append('.IP "' + manify(txt) + '"\n') st.dt_from = None elif txt != '': st.man_out.append(manify(txt) + "\n") elif tag == 'li': if st.list_state[-1] == 'dl': if st.at_first_tag_in_li: die("Invalid 0. -> td translation") tag = 'dd' if txt != '': st.man_out.append(manify(txt) + "\n") st.at_first_tag_in_li = False elif tag == 'blockquote': st.man_out.append(".RE\n") elif tag == 'pre': st.in_pre = False st.man_out.append(manify(txt) + "\n.fi\n") elif (tag == 'code' and not st.in_pre): st.in_code = False add_to_txt = NORM_FONT[0] elif tag == 'strong' or tag == 'b': add_to_txt = NORM_FONT[0] elif tag == 'em' or tag == 'i': tag = 'u' # Change it into underline to be more like the man page add_to_txt = NORM_FONT[0] elif tag == 'ol' or tag == 'ul': if st.list_state.pop() == 'dl': tag = 'dl' if st.list_state: st.man_out.append(".RE\n") else: st.p_macro = ".P\n" st.at_first_tag_in_dd = False st.html_out.append('') if add_to_txt: if txt is None: st.txt += add_to_txt else: txt += add_to_txt if st.dt_from == tag: st.man_out.append('.IP "' + manify(txt) + '"\n') st.html_out.append('
') st.at_first_tag_in_dd = True st.dt_from = None elif tag == 'dt': st.html_out.append('
') st.at_first_tag_in_dd = True def handle_data(self, txt): st = self.state if args.debug: self.output_debug('DATA', (txt,)) if st.in_pre: html = htmlify(txt) else: txt = re.sub(r'\s--(\s)', NBR_SPACE[0] + r'--\1', txt).replace('--', NBR_DASH[0]*2) txt = re.sub(r'(^|\W)-', r'\1' + NBR_DASH[0], txt) html = htmlify(txt) if st.in_code: txt = re.sub(r'\s', NBR_SPACE[0], txt) html = html.replace(NBR_DASH[0], '-').replace(NBR_SPACE[0], ' ') # is non-breaking in CSS st.html_out.append(html.replace(NBR_SPACE[0], ' ').replace(NBR_DASH[0], '-⁠')) st.txt += txt def output_debug(self, event, extra): import pprint st = self.state if args.debug < 2: st = argparse.Namespace(**vars(st)) if len(st.html_out) > 2: st.html_out = ['...'] + st.html_out[-2:] if len(st.man_out) > 2: st.man_out = ['...'] + st.man_out[-2:] print(event, extra) pprint.PrettyPrinter(indent=2).pprint(vars(st)) def manify(txt): return re.sub(r"^(['.])", r'\&\1', txt.replace('\\', '\\\\') .replace(NBR_SPACE[0], NBR_SPACE[1]) .replace(NBR_DASH[0], NBR_DASH[1]) .replace(NORM_FONT[0], NORM_FONT[1]) .replace(BOLD_FONT[0], BOLD_FONT[1]) .replace(UNDR_FONT[0], UNDR_FONT[1]), flags=re.M) def htmlify(txt): return txt.replace('&', '&').replace('<', '<').replace('>', '>').replace('"', '"') def warn(*msg): print(*msg, file=sys.stderr) def die(*msg): warn(*msg) sys.exit(1) if __name__ == '__main__': parser = argparse.ArgumentParser(description='Transform a NAME.NUM.md markdown file into a NAME.NUM.html web page & a NAME.NUM man page.', add_help=False) parser.add_argument('--test', action='store_true', help='Test if we can parse the input w/o updating any files.') parser.add_argument('--debug', '-D', action='count', default=0, help='Output copious info on the html parsing. Repeat for even more.') parser.add_argument("--help", "-h", action="help", help="Output this help message and exit.") parser.add_argument('mdfile', help="The NAME.NUM.md file to parse.") args = parser.parse_args() try: import cmarkgfm md_parser = cmarkgfm.markdown_to_html except: try: import commonmark md_parser = html_via_commonmark except: die("Failed to find cmarkgfm or commonmark for python3.") main()