From 4e93d90e6d8ef9f392225a8f2688b5c42430f4fb Mon Sep 17 00:00:00 2001 From: Oscar Benjamin Date: Thu, 22 Oct 2026 20:51:40 +0100 Subject: [PATCH] Support FLINT's new doc parser --- bin/all_rst_to_pxd.sh | 13 +-- bin/rst_to_pxd.py | 131 +++++++++++++++++++------ src/flint/flintlib/functions/flint.pxd | 2 +- src/flint/types/_gr.pxd | 2 - 4 files changed, 111 insertions(+), 37 deletions(-) diff --git a/bin/all_rst_to_pxd.sh b/bin/all_rst_to_pxd.sh index 091d755e..f05839e8 100755 --- a/bin/all_rst_to_pxd.sh +++ b/bin/all_rst_to_pxd.sh @@ -4,11 +4,12 @@ set -e if [ $# -eq 0 ] then - echo "Usage: bin/all_rst_to_pxd.sh /path/to/flint/doc/source" + echo "Usage: bin/all_rst_to_pxd.sh /path/to/flint/doc/source [parser options]" exit 1 fi FLINT_DOC_DIR=$1 +shift modules=( "flint" @@ -18,7 +19,6 @@ modules=( "gr" "gr_domains" "gr_generic" - "gr_implementing" "gr_mat" "gr_mpoly" "gr_poly" @@ -146,9 +146,10 @@ modules=( # "mpfr_vec" ) -for module in ${modules[@]}; do +for module in "${modules[@]}"; do echo "Processing $module" - bin/rst_to_pxd.py flint/$module \ - --flint-doc-dir=$FLINT_DOC_DIR \ - > src/flint/flintlib/functions/$module.pxd + bin/rst_to_pxd.py "flint/$module" \ + "$@" \ + --flint-doc-dir="$FLINT_DOC_DIR" \ + > "src/flint/flintlib/functions/$module.pxd" done diff --git a/bin/rst_to_pxd.py b/bin/rst_to_pxd.py index 0168f9b7..a10c41aa 100755 --- a/bin/rst_to_pxd.py +++ b/bin/rst_to_pxd.py @@ -25,6 +25,10 @@ will output a skeleton fmpz_poly.pxd to stdout +For FLINT's Markdown documentation, add --use-flint-parser and point +--flint-doc-dir at the checkout's doc/src directory. The parser is loaded +from the adjacent doc/fdoc package. Without the flag, RST parsing is unchanged. + You will need to configure the location of the flintlib submodule directory and the locations of the flint and arb doc source directories. @@ -39,6 +43,8 @@ # recognize a function definition in rst is_func = re.compile(r"\.\.( )+(c:)?function( )*::") +is_macro = re.compile(r"\.\.( )+(c:)?macro( )*::") +typed_macro = re.compile(r"^.+[\s*][A-Za-z_]\w*\s*\(") # rename types to avoid python -- c name collisions rename_types = [ (re.compile(r"\bfmpz\b"),"fmpz_struct"), @@ -54,11 +60,19 @@ import_dict = {} # gr_domains.rst lists functions that are in gr.h doc_to_header = {'flint/gr_domains': 'flint/gr'} -void_parameter_list = re.compile(r"\(\s*void\s*\)$") +void_parameter_list = re.compile(r"\(\s*void\s*\)") macro_declarations = { 'flint/arf': { 'ARF_PREC_EXACT': 'cdef const slong ARF_PREC_EXACT', }, + 'flint/fmpq': { + 'fmpq_numref': 'fmpz * fmpq_numref(const fmpq_t x)', + 'fmpq_denref': 'fmpz * fmpq_denref(const fmpq_t x)', + }, + 'flint/fmpq_poly': { + 'fmpq_poly_numref': 'fmpz * fmpq_poly_numref(fmpq_poly_t poly)', + 'fmpq_poly_denref': 'fmpz * fmpq_poly_denref(fmpq_poly_t poly)', + }, } @@ -127,6 +141,7 @@ def clean_types(function): ret = function.strip() for old, new in rename_types: ret = re.sub(old, new, ret) + # This also applies to callback signatures nested in parameter lists. # Cython expects zero-argument functions to use () rather than (void). ret = re.sub(void_parameter_list, "()", ret) return ret @@ -142,8 +157,9 @@ def get_functions(file): for line in file: # Keep track of the macros # We want to give them types in cython... - if line.startswith('.. macro'): + if is_macro.match(line): macros.append(line.strip()) + in_list = False continue if line.strip().endswith('\\'): @@ -162,6 +178,29 @@ def get_functions(file): return ret, macros +def get_macro_declarations(macros, h_name): + """Use documented types or explicit overrides; never infer macro types. + + Cython can call a function-like C macro using a function declaration. + Untyped macros (which may also introduce variables or require lvalues) + cannot in general be represented this way. + """ + functions, constants = [], [] + overrides = macro_declarations.get(h_name, {}) + for macro in macros: + signature = macro.split('::', 1)[1].strip() + name = signature.split('(', 1)[0].strip() + declaration = overrides.get(name) + if declaration is not None: + if declaration.startswith('cdef '): + constants.append(declaration) + else: + functions.append(clean_types(declaration)) + elif typed_macro.match(signature): + functions.append(clean_types(signature)) + return functions, constants + + def get_all_types(function_list): ret = set() for f in function_list: @@ -170,6 +209,32 @@ def get_all_types(function_list): return ret +def get_flint_functions(docdir, name): + """Read a Markdown module using the parser from its FLINT checkout. + + docdir is FLINT's doc/src directory; fdoc lives alongside it. + """ + docdir = Path(docdir).resolve() + sys.path.insert(0, str(docdir.parent)) + try: + from fdoc.markup import BlockParser, split_front_matter + finally: + sys.path.pop(0) + + path = docdir / name / 'index.md' + metadata, body, line = split_front_matter(path.read_text()) + nodes = BlockParser(path=str(path)).parse(body, line) + functions, macros = [], [] + for node in nodes: + if node.kind != 'div': + continue + if node['kind'] == 'function': + functions.extend(clean_types(p) for p in node['protos']) + elif node['kind'] == 'macro': + macros.extend('.. macro:: ' + p for p in node['protos']) + return functions, macros, metadata.get('header') + + def has_types(line, types): #ts = set(t for t in get_parameter_types(line) if t in types) #if ts: @@ -208,38 +273,45 @@ def generate_pxd_file(h_name, opts): if name[:6] == "flint/": docdir = opts.flint_doc_dir name = name[6:] - with open(os.path.join(docdir, name + ".rst")) as f: - l, macros = get_functions(f) - unknown_types = gen_imports(l) - macro_defs = [] - declared_macros = macro_declarations.get(h_name, {}) - for macro in macros: - macro_name = macro.split("::", 1)[1].strip() - if macro_name in declared_macros: - macro_defs.append(declared_macros[macro_name]) - print() - for t in unknown_types: - print("# unknown type " + t) - print() - for m in macros: - print("# " + m) - print() - print(r'cdef extern from "' + h_name + r'.h":') - for declaration in macro_defs: - print(" " + declaration) - for f in l: - if has_types(f, unknown_types): - print(" # " + f) - else: - print(" " + f) + if opts.use_flint_parser: + l, macros, header = get_flint_functions(docdir, name) + if header: + h_name = 'flint/' + header.removesuffix('.h') + else: + with open(os.path.join(docdir, name + ".rst")) as f: + l, macros = get_functions(f) + + macro_functions, macro_defs = get_macro_declarations(macros, h_name) + l.extend(macro_functions) + if opts.use_flint_parser and name == 'gr_domains': + # Context predicates moved from gr_domains to gr in the Markdown docs. + # Preserve the import path used by bindings generated from older docs. + print('from flint.flintlib.functions.gr cimport *') + unknown_types = gen_imports(l) + print() + for t in unknown_types: + print("# unknown type " + t) + print() + for m in macros: + print("# " + m) + print() + print(r'cdef extern from "' + h_name + r'.h":') + for declaration in macro_defs: + print(" " + declaration) + for f in l: + if has_types(f, unknown_types): + print(" # " + f) + else: + print(" " + f) def main(*args): usage = """ $ cd /path/to/python-flint $ bin/rst_to_pxd.py flint/fmpz --flint-doc-dir=/path/to/flint/doc/source + $ bin/rst_to_pxd.py flint/fmpz --use-flint-parser --flint-doc-dir=/path/to/flint/doc/src """ - parser = argparse.ArgumentParser(description='Generate a pxd file from an rst file', + parser = argparse.ArgumentParser(description='Generate a pxd file from FLINT documentation', usage=usage) parser.add_argument('--flint-lib-dir', help='location of the flintlib submodule', default="./src/flint/flintlib") @@ -247,7 +319,10 @@ def main(*args): default="/Users/davideinstein/projects/arb/doc/source") parser.add_argument('--flint-doc-dir', help='location of the flint doc source directory', default="/Users/davideinstein/projects/flint2/doc/source") - parser.add_argument('name', help='name of the rst file to parse (e.g. flint/fmpz)') + parser.add_argument('--use-flint-parser', action='store_true', + help="parse Markdown using FLINT's doc/fdoc parser; " + "--flint-doc-dir must point to FLINT's doc/src") + parser.add_argument('name', help='documentation module to parse (e.g. flint/fmpz)') args = parser.parse_args() diff --git a/src/flint/flintlib/functions/flint.pxd b/src/flint/flintlib/functions/flint.pxd index 8c8e1900..bb6cde67 100644 --- a/src/flint/flintlib/functions/flint.pxd +++ b/src/flint/flintlib/functions/flint.pxd @@ -45,7 +45,7 @@ cdef extern from "flint/flint.h": # int flint_sscanf(const char * s, const char * str, ...) void flint_abort() # void flint_throw(flint_err_t exc, const char * msg, ...) - # void flint_set_abort(void (* func)(void)) + # void flint_set_abort(void (* func)()) # void flint_set_throw(void (* func)(flint_err_t, const char *, va_list)) # void flint_merge_sort(void * buf, slong len, slong size, int (* cmp) (const void *, const void *, void *), void * data) # void flint_sort(void * buf, slong len, slong size, int (* cmp) (const void *, const void *, void *), void * data) diff --git a/src/flint/types/_gr.pxd b/src/flint/types/_gr.pxd index e53187f5..683d7a28 100644 --- a/src/flint/types/_gr.pxd +++ b/src/flint/types/_gr.pxd @@ -118,8 +118,6 @@ from flint.flintlib.functions.gr cimport ( gr_is_zero, gr_is_one, gr_is_neg_one, - gr_is_integer, - gr_is_rational, gr_equal,