diff --git a/Documentation/tools/kdoc_ancillary.rst b/Documentation/tools/kdoc_ancillary.rst index 3950d0a3f104..85f3806a431a 100644 --- a/Documentation/tools/kdoc_ancillary.rst +++ b/Documentation/tools/kdoc_ancillary.rst @@ -21,6 +21,15 @@ Regular expression class handler :undoc-members: +C tokenizer +=========== + +.. automodule:: lib.python.kdoc.c_lex + :members: + :show-inheritance: + :undoc-members: + + Chinese, Japanese and Korean variable fonts handler =================================================== diff --git a/tools/docs/kernel-doc b/tools/docs/kernel-doc index 3a932f95bdf5..d9192c3f1645 100755 --- a/tools/docs/kernel-doc +++ b/tools/docs/kernel-doc @@ -240,11 +240,9 @@ def main(): help=EXPORT_FILE_DESC) # - # Output format mutually-exclusive group + # Output format # - out_group = parser.add_argument_group("Output format selection (mutually exclusive)") - - out_fmt = out_group.add_mutually_exclusive_group() + out_fmt = parser.add_argument_group("Output format selection (mutually exclusive)") out_fmt.add_argument("-m", "-man", "--man", action="store_true", help="Output troff manual page format.") @@ -253,6 +251,12 @@ def main(): out_fmt.add_argument("-N", "-none", "--none", action="store_true", help="Do not output documentation, only warnings.") + out_fmt.add_argument("-y", "--yaml-file", "--yaml", + help="Stores kernel-doc output on a yaml file.") + out_fmt.add_argument("-k", "--kdoc-item", "--kdoc", action="store_true", + help="Store KdocItem inside yaml file. Ued together with --yaml.") + + # # Output selection mutually-exclusive group # @@ -323,14 +327,42 @@ def main(): from kdoc.kdoc_files import KernelFiles # pylint: disable=C0415 from kdoc.kdoc_output import RestFormat, ManFormat # pylint: disable=C0415 - if args.man: - out_style = ManFormat(modulename=args.modulename) - elif args.none: + yaml_content = set() + if args.yaml_file: out_style = None + + if args.man: + yaml_content |= {"man"} + + if args.rst: + yaml_content |= {"rst"} + + if args.kdoc_item or not yaml_content: + yaml_content |= {"KdocItem"} + else: - out_style = RestFormat() + n_outputs = 0 + + if args.man: + out_style = ManFormat(modulename=args.modulename) + n_outputs += 1 + + if args.none: + out_style = None + n_outputs += 1 + + if args.rst or n_outputs == 0: + n_outputs += 1 + out_style = RestFormat() + + if n_outputs > 1: + parser.error("Those arguments are muttually exclusive: --man, --rst, --none, except when generating a YAML file.") + + elif not n_outputs: + out_style = RestFormat() kfiles = KernelFiles(verbose=args.verbose, + yaml_file=args.yaml_file, yaml_content=yaml_content, out_style=out_style, werror=args.werror, wreturn=args.wreturn, wshort_desc=args.wshort_desc, wcontents_before_sections=args.wcontents_before_sections) diff --git a/tools/lib/python/kdoc/c_lex.py b/tools/lib/python/kdoc/c_lex.py index b6d58bd470a9..e01b154f458e 100644 --- a/tools/lib/python/kdoc/c_lex.py +++ b/tools/lib/python/kdoc/c_lex.py @@ -336,13 +336,14 @@ class CTokenArgs: self.sub_tokeninzer = CTokenizer(sub_str) def groups(self, new_tokenizer): - """ + r""" Create replacement arguments for backrefs like: - ``\0``, ``\1``, ``\2``, ...``\n`` + ``\0``, ``\1``, ``\2``, ... ``\{number}`` - It also accepts a ``+`` character to the highest backref. When used, - it means in practice to ignore delimins after it, being greedy. + It also accepts a ``+`` character to the highest backref, like + ``\4+``. When used, the backref will be greedy, picking all other + arguments afterwards. The logic is smart enough to only go up to the maximum required argument, even if there are more. diff --git a/tools/lib/python/kdoc/kdoc_files.py b/tools/lib/python/kdoc/kdoc_files.py index 8c2059623949..5a299ed44d62 100644 --- a/tools/lib/python/kdoc/kdoc_files.py +++ b/tools/lib/python/kdoc/kdoc_files.py @@ -9,7 +9,6 @@ Classes for navigating through the files that kernel-doc needs to handle to generate documentation. """ -import argparse import logging import os import re @@ -17,6 +16,7 @@ import re from kdoc.kdoc_parser import KernelDoc from kdoc.xforms_lists import CTransforms from kdoc.kdoc_output import OutputFormat +from kdoc.kdoc_yaml_file import KDocTestFile class GlobSourceFiles: @@ -87,6 +87,28 @@ class GlobSourceFiles: file_not_found_cb(fname) +class KdocConfig(): + """ + Stores all configuration attributes that kdoc_parser and kdoc_output + needs. + """ + def __init__(self, verbose=False, werror=False, wreturn=False, + wshort_desc=False, wcontents_before_sections=False, + logger=None): + + self.verbose = verbose + self.werror = werror + self.wreturn = wreturn + self.wshort_desc = wshort_desc + self.wcontents_before_sections = wcontents_before_sections + + if logger: + self.log = logger + else: + self.log = logging.getLogger(__file__) + + self.warning = self.log.warning + class KernelFiles(): """ Parse kernel-doc tags on multiple kernel source files. @@ -131,6 +153,12 @@ class KernelFiles(): If not specified, defaults to use: ``logging.getLogger("kernel-doc")`` + ``yaml_file`` + If defined, stores the output inside a YAML file. + + ``yaml_content`` + Defines what will be inside the YAML file. + Note: There are two type of parsers defined here: @@ -160,7 +188,12 @@ class KernelFiles(): if fname in self.files: return - doc = KernelDoc(self.config, fname, self.xforms) + if self.test_file: + store_src = True + else: + store_src = False + + doc = KernelDoc(self.config, fname, self.xforms, store_src=store_src) export_table, entries = doc.parse_kdoc() self.export_table[fname] = export_table @@ -170,6 +203,10 @@ class KernelFiles(): self.results[fname] = entries + source = doc.get_source() + if source: + self.source[fname] = source + def process_export_file(self, fname): """ Parses ``EXPORT_SYMBOL*`` macros from a single Kernel source file. @@ -199,7 +236,7 @@ class KernelFiles(): def __init__(self, verbose=False, out_style=None, xforms=None, werror=False, wreturn=False, wshort_desc=False, wcontents_before_sections=False, - logger=None): + yaml_file=None, yaml_content=None, logger=None): """ Initialize startup variables and parse all files. """ @@ -224,37 +261,40 @@ class KernelFiles(): if kdoc_werror: werror = kdoc_werror + if not logger: + logger = logging.getLogger("kernel-doc") + else: + logger = logger + # Some variables are global to the parser logic as a whole as they are # used to send control configuration to KernelDoc class. As such, # those variables are read-only inside the KernelDoc. - self.config = argparse.Namespace + self.config = KdocConfig(verbose, werror, wreturn, wshort_desc, + wcontents_before_sections, logger) - self.config.verbose = verbose - self.config.werror = werror - self.config.wreturn = wreturn - self.config.wshort_desc = wshort_desc - self.config.wcontents_before_sections = wcontents_before_sections + # Override log warning, as we want to count errors + self.config.warning = self.warning + + if yaml_file: + self.test_file = KDocTestFile(self.config, yaml_file, yaml_content) + else: + self.test_file = None if xforms: self.xforms = xforms else: self.xforms = CTransforms() - if not logger: - self.config.log = logging.getLogger("kernel-doc") - else: - self.config.log = logger - - self.config.warning = self.warning - self.config.src_tree = os.environ.get("SRCTREE", None) # Initialize variables that are internal to KernelFiles self.out_style = out_style + self.out_style.set_config(self.config) self.errors = 0 self.results = {} + self.source = {} self.files = set() self.export_files = set() @@ -294,8 +334,6 @@ class KernelFiles(): returning kernel-doc markups on each interaction. """ - self.out_style.set_config(self.config) - if not filenames: filenames = sorted(self.results.keys()) @@ -315,29 +353,29 @@ class KernelFiles(): for s in symbol: function_table.add(s) - self.out_style.set_filter(export, internal, symbol, nosymbol, - function_table, enable_lineno, - no_doc_sections) - - msg = "" if fname not in self.results: self.config.log.warning("No kernel-doc for file %s", fname) continue symbols = self.results[fname] - self.out_style.set_symbols(symbols) - for arg in symbols: - m = self.out_msg(fname, arg.name, arg) + if self.test_file: + self.test_file.set_filter(export, internal, symbol, nosymbol, + function_table, enable_lineno, + no_doc_sections) - if m is None: - ln = arg.get("ln", 0) - dtype = arg.get('type', "") + self.test_file.output_symbols(fname, symbols, + self.source.get(fname)) - self.config.log.warning("%s:%d Can't handle %s", - fname, ln, dtype) - else: - msg += m + continue + self.out_style.set_filter(export, internal, symbol, nosymbol, + function_table, enable_lineno, + no_doc_sections) + + msg = self.out_style.output_symbols(fname, symbols) if msg: yield fname, msg + + if self.test_file: + self.test_file.write() diff --git a/tools/lib/python/kdoc/kdoc_item.py b/tools/lib/python/kdoc/kdoc_item.py index 2b8a93f79716..fe08cac861c2 100644 --- a/tools/lib/python/kdoc/kdoc_item.py +++ b/tools/lib/python/kdoc/kdoc_item.py @@ -22,15 +22,34 @@ class KdocItem: self.sections = {} self.sections_start_lines = {} self.parameterlist = [] - self.parameterdesc_start_lines = [] + self.parameterdesc_start_lines = {} self.parameterdescs = {} self.parametertypes = {} + + self.warnings = [] + # # Just save everything else into our own dict so that the output # side can grab it directly as before. As we move things into more # structured data, this will, hopefully, fade away. # - self.other_stuff = other_stuff + known_keys = { + 'declaration_start_line', + 'sections', + 'sections_start_lines', + 'parameterlist', + 'parameterdesc_start_lines', + 'parameterdescs', + 'parametertypes', + 'warnings', + } + + self.other_stuff = {} + for k, v in other_stuff.items(): + if k in known_keys: + setattr(self, k, v) # real attribute + else: + self.other_stuff[k] = v def get(self, key, default = None): """ @@ -41,6 +60,20 @@ class KdocItem: def __getitem__(self, key): return self.get(key) + @classmethod + def from_dict(cls, d): + """Create a KdocItem from a plain dict.""" + + cp = d.copy() + name = cp.pop('name', None) + fname = cp.pop('fname', None) + type = cp.pop('type', None) + start_line = cp.pop('start_line', 1) + other_stuff = cp.pop('other_stuff', {}) + + # Everything that’s left goes straight to __init__ + return cls(name, fname, type, start_line, **cp, **other_stuff) + # # Tracking of section and parameter information. # @@ -49,7 +82,7 @@ class KdocItem: Set sections and start lines. """ self.sections = sections - self.section_start_lines = start_lines + self.sections_start_lines = start_lines def set_params(self, names, descs, types, starts): """ diff --git a/tools/lib/python/kdoc/kdoc_output.py b/tools/lib/python/kdoc/kdoc_output.py index 08539dd92cbb..1b54117dbe19 100644 --- a/tools/lib/python/kdoc/kdoc_output.py +++ b/tools/lib/python/kdoc/kdoc_output.py @@ -222,6 +222,27 @@ class OutputFormat: return None + def output_symbols(self, fname, symbols): + """ + Handles a set of KdocItem symbols. + """ + self.set_symbols(symbols) + + msg = "" + for arg in symbols: + m = self.msg(fname, arg.name, arg) + + if m is None: + ln = arg.get("ln", 0) + dtype = arg.get('type', "") + + self.config.log.warning("%s:%d Can't handle %s", + fname, ln, dtype) + else: + msg += m + + return msg + # Virtual methods to be overridden by inherited classes # At the base class, those do nothing. def set_symbols(self, symbols): @@ -368,7 +389,7 @@ class RestFormat(OutputFormat): else: self.data += f'{self.lineprefix}**{section}**\n\n' - self.print_lineno(args.section_start_lines.get(section, 0)) + self.print_lineno(args.sections_start_lines.get(section, 0)) self.output_highlight(text) self.data += "\n" self.data += "\n" diff --git a/tools/lib/python/kdoc/kdoc_parser.py b/tools/lib/python/kdoc/kdoc_parser.py index f6c4ee3b18c9..a10e64589d76 100644 --- a/tools/lib/python/kdoc/kdoc_parser.py +++ b/tools/lib/python/kdoc/kdoc_parser.py @@ -140,7 +140,7 @@ class KernelEntry: self.parametertypes = {} self.parameterdesc_start_lines = {} - self.section_start_lines = {} + self.sections_start_lines = {} self.sections = {} self.anon_struct_union = False @@ -220,7 +220,7 @@ class KernelEntry: self.sections[name] += '\n' + contents else: self.sections[name] = contents - self.section_start_lines[name] = self.new_start_line + self.sections_start_lines[name] = self.new_start_line self.new_start_line = 0 # self.config.log.debug("Section: %s : %s", name, pformat(vars(self))) @@ -246,12 +246,13 @@ class KernelDoc: #: String to write when a parameter is not described. undescribed = "-- undescribed --" - def __init__(self, config, fname, xforms): + def __init__(self, config, fname, xforms, store_src=False): """Initialize internal variables""" self.fname = fname self.config = config self.xforms = xforms + self.store_src = store_src tokenizer_set_log(self.config.log, f"{self.fname}: CMatch: ") @@ -264,6 +265,9 @@ class KernelDoc: # Place all potential outputs into an array self.entries = [] + # When store_src is true, the kernel-doc source content is stored here + self.source = None + # # We need Python 3.7 for its "dicts remember the insertion # order" guarantee @@ -316,7 +320,7 @@ class KernelDoc: for section in ["Description", "Return"]: if section in sections and not sections[section].rstrip(): del sections[section] - item.set_sections(sections, self.entry.section_start_lines) + item.set_sections(sections, self.entry.sections_start_lines) item.set_params(self.entry.parameterlist, self.entry.parameterdescs, self.entry.parametertypes, self.entry.parameterdesc_start_lines) @@ -1592,6 +1596,15 @@ class KernelDoc: state.DOCBLOCK: process_docblock, } + def get_source(self): + """ + Return the file content of the lines handled by kernel-doc at the + latest parse_kdoc() run. + + Returns none if KernelDoc() was not initialized with store_src, + """ + return self.source + def parse_kdoc(self): """ Open and process each line of a C source file. @@ -1605,6 +1618,8 @@ class KernelDoc: prev = "" prev_ln = None export_table = set() + self.source = [] + self.state = state.NORMAL try: with open(self.fname, "r", encoding="utf8", @@ -1631,6 +1646,8 @@ class KernelDoc: ln, state.name[self.state], line) + prev_state = self.state + # This is an optimization over the original script. # There, when export_file was used for the same file, # it was read twice. Here, we use the already-existing @@ -1641,6 +1658,14 @@ class KernelDoc: # Hand this line to the appropriate state handler self.state_actions[self.state](self, ln, line) + if self.store_src and prev_state != self.state or self.state != state.NORMAL: + if self.state == state.NAME: + # A "/**" was detected. Add a new source element + self.source.append({"ln": ln, "data": line + "\n"}) + else: + # Append to the existing one + self.source[-1]["data"] += line + "\n" + self.emit_unused_warnings() except OSError: diff --git a/tools/lib/python/kdoc/kdoc_re.py b/tools/lib/python/kdoc/kdoc_re.py index 6f3ae28859ea..28292efe25a2 100644 --- a/tools/lib/python/kdoc/kdoc_re.py +++ b/tools/lib/python/kdoc/kdoc_re.py @@ -70,10 +70,15 @@ class KernRe: flags_name = " | ".join(flags) + max_len = 60 + pattern = "" + for pos in range(0, len(self.regex.pattern), max_len): + pattern += '"' + self.regex.pattern[pos:max_len + pos] + '" ' + if flags_name: - return f'KernRe("{self.regex.pattern}", {flags_name})' + return f'KernRe({pattern}, {flags_name})' else: - return f'KernRe("{self.regex.pattern}")' + return f'KernRe({pattern})' def __add__(self, other): """ diff --git a/tools/lib/python/kdoc/kdoc_yaml_file.py b/tools/lib/python/kdoc/kdoc_yaml_file.py new file mode 100644 index 000000000000..db131503c3f6 --- /dev/null +++ b/tools/lib/python/kdoc/kdoc_yaml_file.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2026: Mauro Carvalho Chehab . + +import os + +from kdoc.kdoc_output import ManFormat, RestFormat + + +class KDocTestFile(): + """ + Handles the logic needed to store kernel‑doc output inside a YAML file. + Useful for unit tests and regression tests. + """ + + def __init__(self, config, yaml_file, yaml_content): + # + # Bail out early if yaml is not available + # + try: + import yaml + except ImportError: + sys.exit("Warning: yaml package not available. Aborting it.") + + self.config = config + self.test_file = os.path.expanduser(yaml_file) + self.yaml_content = yaml_content + + self.tests = [] + + out_dir = os.path.dirname(self.test_file) + if out_dir and not os.path.isdir(out_dir): + sys.exit(f"Directory {out_dir} doesn't exist.") + + self.out_style = [] + + if "man" in self.yaml_content: + out_style = ManFormat() + out_style.set_config(self.config) + + self.out_style.append(out_style) + + if "rst" in self.yaml_content: + out_style = RestFormat() + out_style.set_config(self.config) + + self.out_style.append(out_style) + + def set_filter(self, export, internal, symbol, nosymbol, + function_table, enable_lineno, no_doc_sections): + """ + Set filters at the output classes. + """ + for out_style in self.out_style: + out_style.set_filter(export, internal, symbol, + nosymbol, function_table, + enable_lineno, no_doc_sections) + + @staticmethod + def get_kdoc_item(arg, start_line=1): + + d = vars(arg) + + declaration_start_line = d.get("declaration_start_line") + if not declaration_start_line: + return d + + d["declaration_start_line"] = start_line + + parameterdesc_start_lines = d.get("parameterdesc_start_lines") + if parameterdesc_start_lines: + for key in parameterdesc_start_lines: + ln = parameterdesc_start_lines[key] + ln += start_line - declaration_start_line + + parameterdesc_start_lines[key] = ln + + sections_start_lines = d.get("sections_start_lines") + if sections_start_lines: + for key in sections_start_lines: + ln = sections_start_lines[key] + ln += start_line - declaration_start_line + + sections_start_lines[key] = ln + + return d + + def output_symbols(self, fname, symbols, source): + """ + Store source, symbols and output strings at self.tests. + """ + + # + # KdocItem needs to be converted into dicts + # + kdoc_item = [] + expected = [] + + if not symbols and not source: + return + + if not source or len(symbols) != len(source): + print(f"Warning: lengths are different. Ignoring {fname}") + + # Folding without line numbers is too hard. + # The right thing to do here to proceed would be to delete + # not-handled source blocks, as len(source) should be bigger + # than len(symbols) + return + + base_name = "test_" + fname.replace(".", "_").replace("/", "_") + expected_dict = {} + start_line=1 + + for i in range(0, len(symbols)): + arg = symbols[i] + + if "KdocItem" in self.yaml_content: + msg = self.get_kdoc_item(arg) + + expected_dict["kdoc_item"] = msg + + for out_style in self.out_style: + if isinstance(out_style, ManFormat): + key = "man" + else: + key = "rst" + + expected_dict[key]= out_style.output_symbols(fname, [arg]) + + name = f"{base_name}_{i:03d}" + + test = { + "name": name, + "description": f"{fname} line {source[i]["ln"]}", + "fname": fname, + "source": source[i]["data"], + "expected": [expected_dict] + } + + self.tests.append(test) + + expected_dict = {} + + def write(self): + """ + Output the content of self.tests to self.test_file. + """ + import yaml + + data = {"tests": self.tests} + + with open(self.test_file, "w", encoding="utf-8") as fp: + yaml.safe_dump(data, fp, sort_keys=False, default_style="|", + default_flow_style=False, allow_unicode=True) diff --git a/tools/unittests/kdoc-test-schema.yaml b/tools/unittests/kdoc-test-schema.yaml new file mode 100644 index 000000000000..cf5079711cd8 --- /dev/null +++ b/tools/unittests/kdoc-test-schema.yaml @@ -0,0 +1,156 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2026: Mauro Carvalho Chehab . + +# KDoc Test File Schema + +# This schema contains objects and properties needed to run kernel-doc +# self-tests. + +$schema: "http://json-schema.org/draft-07/schema#" + +tests: + type: array + minItems: 1 + description: | + A list of kernel-doc tests. + + properties: + type: object + properties: + name: + type: string + description: | + Test name. Should be an unique identifier within the schema. + Don't prepend it with "test", as the dynamic test creation will + do it. + + description: + type: string + description: | + Test description + + source: + type: string + description: | + C source code that should be parsed by kernel-doc. + + fname: + type: string + description: | + The filename that contains the element. + When placing real testcases, please use here the name of + the C file (or header) from where the source code was picked. + + exports: + type: array + items: { type: string } + description: | + A list of export identifiers that are expected when parsing source. + + expected: + type: array + minItems: 1 + description: | + A list of expected values. This list consists on objects to check + both kdoc_parser and/or kdoc_output objects. + + items: + type: object + properties: + # + # kdoc_item + # + kdoc_item: + type: object + description: | + Object expected to represent the C source code after parsed + by tools/lib/python/kdoc/kdoc_parser.py KernelDoc class. + See tools/lib/python/kdoc/kdoc_item.py for its contents. + + properties: + name: + type: string + description: | + The name of the identifier (function name, struct name, etc). + type: + type: string + description: | + Type of the object, as filled by kdoc_parser. can be: + - enum + - typedef + - union + - struct + - var + - function + declaration_start_line: + type: integer + description: | + The line number where the kernel-doc markup started. + The first line of the code is line number 1. + sections: + type: object + additionalProperties: { type: string } + description: | + Sections inside the kernel-doc markups: + - "description" + - "return" + - any other part of the markup that starts with "something:" + sections_start_lines: + type: object + additionalProperties: { type: integer } + description: | + a list of section names and the starting line of it. + parameterlist: + type: array + items: { type: string } + description: | + Ordered list of parameter names. + + parameterdesc_start_lines: + type: object + additionalProperties: { type: integer } + description: | + Mapping from parameter name to the line where its + description starts. + parameterdescs: + type: object + additionalProperties: { type: string } + description: | + Mapping from parameter name to its description. + + parametertypes: + type: object + additionalProperties: { type: string } + description: | + Mapping from parameter name to its type. + + other_stuff: + type: object + additionalProperties: {} + description: | + Extra properties that will be stored at the item. + Should match what kdoc_output expects. + + required: + - name + - type + - declaration_start_line + + rst: + type: string + description: | + The expected output for RestOutput class. + + man: + type: string + description: | + The expected output for ManOutput class. + + anyOf: + required: kdoc_item + required: source + + required: + - name + - fname + - expected diff --git a/tools/unittests/kdoc-test.yaml b/tools/unittests/kdoc-test.yaml new file mode 100644 index 000000000000..b6e04f10ccdb --- /dev/null +++ b/tools/unittests/kdoc-test.yaml @@ -0,0 +1,154 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (c) 2026: Mauro Carvalho Chehab + +# Test cases for the dynamic tests. +# Useful to test if kernel-doc classes are doing what it is expected. +# + +tests: +- name: func1 + fname: mock_functions.c + description: "Simplest function test: do nothing, just rst output" + + source: | + /** + * func1 - Not exported function + * @arg1: @arg1 does nothing + * + * Does nothing + * + * return: + * always return 0. + */ + int func1(char *arg1) { return 0; }; + + + expected: + - rst: | + .. c:function:: int func1 (char *arg1) + + Not exported function + + .. container:: kernelindent + + **Parameters** + + ``char *arg1`` + **arg1** does nothing + + **Description** + + Does nothing + + **Return** + + always return 0. + + # TODO: how to handle timestamps at .TH? + man: | + .TH "func1" 9 "February 2026" "" "Kernel API Manual" + .SH NAME + func1 \- Not exported function + .SH SYNOPSIS + .B "int" func1 + .BI "(char *arg1 " ");" + .SH ARGUMENTS + .IP "arg1" 12 + \fIarg1\fP does nothing + .SH "DESCRIPTION" + Does nothing + .SH "RETURN" + always return 0. + .SH "SEE ALSO" + .PP + Kernel file \fBmock_functions.c\fR + +- name: func2 + fname: func2.c + description: Simple test with exports + + source: | + /** + * func2() - Exported function + * @arg1: @arg1 does nothing + * + * Does nothing + * + * return: + * always return 0. + */ + int func2(char *arg1) { return 0; }; + EXPORT_SYMBOL(func2); + + exports: func2 + expected: + - kdoc_item: + name: func2 + type: function + declaration_start_line: 1 + + sections: + Description: | + Does nothing + + Return: | + always return 0. + + sections_start_lines: + Description: 3 + Return: 6 + + parameterdescs: + arg1: | + @arg1 does nothing + parameterlist: + - arg1 + parameterdesc_start_lines: + arg1: 2 + parametertypes: + arg1: char *arg1 + + other_stuff: + func_macro: false + functiontype: int + purpose: "Exported function" + typedef: false + + rst: | + .. c:function:: int func2 (char *arg1) + + Exported function + + .. container:: kernelindent + + **Parameters** + + ``char *arg1`` + **arg1** does nothing + + **Description** + + Does nothing + + **Return** + + always return 0. + + # TODO: how to handle timestamps at .TH? + man: | + .TH "func2" 9 "February 2026" "" "Kernel API Manual" + .SH NAME + func2 \- Exported function + .SH SYNOPSIS + .B "int" func2 + .BI "(char *arg1 " ");" + .SH ARGUMENTS + .IP "arg1" 12 + \fIarg1\fP does nothing + .SH "DESCRIPTION" + Does nothing + .SH "RETURN" + always return 0. + .SH "SEE ALSO" + .PP + Kernel file \fBfunc2.c\fR diff --git a/tools/unittests/test_kdoc_parser.py b/tools/unittests/test_kdoc_parser.py new file mode 100755 index 000000000000..723dd8c7f4f3 --- /dev/null +++ b/tools/unittests/test_kdoc_parser.py @@ -0,0 +1,534 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# Copyright(c) 2026: Mauro Carvalho Chehab . +# +# pylint: disable=C0200,C0413,W0102,R0914 + +""" +Unit tests for kernel-doc parser. +""" + +import logging +import os +import re +import shlex +import sys +import unittest + +from textwrap import dedent +from unittest.mock import patch, MagicMock, mock_open + +import yaml + +SRC_DIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(SRC_DIR, "../lib/python")) + +from kdoc.kdoc_files import KdocConfig +from kdoc.kdoc_item import KdocItem +from kdoc.kdoc_parser import KernelDoc +from kdoc.kdoc_output import RestFormat, ManFormat + +from kdoc.xforms_lists import CTransforms + +from unittest_helper import run_unittest + + +# +# Test file +# +TEST_FILE = os.path.join(SRC_DIR, "kdoc-test.yaml") + +# +# Ancillary logic to clean whitespaces +# +#: Regex to help cleaning whitespaces +RE_WHITESPC = re.compile(r"[ \t]++") +RE_BEGINSPC = re.compile(r"^\s+", re.MULTILINE) +RE_ENDSPC = re.compile(r"\s+$", re.MULTILINE) + +def clean_whitespc(val, relax_whitespace=False): + """ + Cleanup whitespaces to avoid false positives. + + By default, strip only bein/end whitespaces, but, when relax_whitespace + is true, also replace multiple whitespaces in the middle. + """ + + if isinstance(val, str): + val = val.strip() + if relax_whitespace: + val = RE_WHITESPC.sub(" ", val) + val = RE_BEGINSPC.sub("", val) + val = RE_ENDSPC.sub("", val) + elif isinstance(val, list): + val = [clean_whitespc(item, relax_whitespace) for item in val] + elif isinstance(val, dict): + val = {k: clean_whitespc(v, relax_whitespace) for k, v in val.items()} + return val + +# +# Helper classes to help mocking with logger and config +# +class MockLogging(logging.Handler): + """ + Simple class to store everything on a list + """ + + def __init__(self, level=logging.NOTSET): + super().__init__(level) + self.messages = [] + self.formatter = logging.Formatter() + + def emit(self, record: logging.LogRecord) -> None: + """ + Append a formatted record to self.messages. + """ + try: + # The `format` method uses the handler's formatter. + message = self.format(record) + self.messages.append(message) + except Exception: + self.handleError(record) + +class MockKdocConfig(KdocConfig): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + + self.log = logging.getLogger(__file__) + self.handler = MockLogging() + self.log.addHandler(self.handler) + + def warning(self, msg): + """Ancillary routine to output a warning and increment error count.""" + + self.log.warning(msg) + +# +# Helper class to generate KdocItem and validate its contents +# +# TODO: check self.config.handler.messages content +# +class GenerateKdocItem(unittest.TestCase): + """ + Base class to run KernelDoc parser class + """ + + DEFAULT = vars(KdocItem("", "", "", 0)) + + config = MockKdocConfig() + xforms = CTransforms() + + def setUp(self): + self.maxDiff = None + + def run_test(self, source, __expected_list, exports={}, fname="test.c", + relax_whitespace=False): + """ + Stores expected values and patch the test to use source as + a "file" input. + """ + debug_level = int(os.getenv("VERBOSE", "0")) + source = dedent(source) + + # Ensure that default values will be there + expected_list = [] + for e in __expected_list: + if not isinstance(e, dict): + e = vars(e) + + new_e = self.DEFAULT.copy() + new_e["fname"] = fname + for key, value in e.items(): + new_e[key] = value + + expected_list.append(new_e) + + patcher = patch('builtins.open', + new_callable=mock_open, read_data=source) + + kernel_doc = KernelDoc(self.config, fname, self.xforms) + + with patcher: + export_table, entries = kernel_doc.parse_kdoc() + + self.assertEqual(export_table, exports) + self.assertEqual(len(entries), len(expected_list)) + + for i in range(0, len(entries)): + + entry = entries[i] + expected = expected_list[i] + self.assertNotEqual(expected, None) + self.assertNotEqual(expected, {}) + self.assertIsInstance(entry, KdocItem) + + d = vars(entry) + for key, value in expected.items(): + result = clean_whitespc(d[key], relax_whitespace) + value = clean_whitespc(value, relax_whitespace) + + if debug_level > 1: + sys.stderr.write(f"{key}: assert('{result}' == '{value}')\n") + + self.assertEqual(result, value, msg=f"at {key}") + +# +# Ancillary function that replicates kdoc_files way to generate output +# +def cleanup_timestamp(text): + lines = text.split("\n") + + for i, line in enumerate(lines): + if not line.startswith('.TH'): + continue + + parts = shlex.split(line) + if len(parts) > 3: + parts[3] = "" + + lines[i] = " ".join(parts) + + + return "\n".join(lines) + +def gen_output(fname, out_style, symbols, expected, + config=None, relax_whitespace=False): + """ + Use the output class to return an output content from KdocItem symbols. + """ + + if not config: + config = MockKdocConfig() + + out_style.set_config(config) + + msg = out_style.output_symbols(fname, symbols) + + result = clean_whitespc(msg, relax_whitespace) + result = cleanup_timestamp(result) + + expected = clean_whitespc(expected, relax_whitespace) + expected = cleanup_timestamp(expected) + + return result, expected + +# +# Classes to be used by dynamic test generation from YAML +# +class CToKdocItem(GenerateKdocItem): + def setUp(self): + self.maxDiff = None + + def run_parser_test(self, source, symbols, exports, fname): + if isinstance(symbols, dict): + symbols = [symbols] + + if isinstance(exports, str): + exports=set([exports]) + elif isinstance(exports, list): + exports=set(exports) + + self.run_test(source, symbols, exports=exports, + fname=fname, relax_whitespace=True) + +class KdocItemToMan(unittest.TestCase): + out_style = ManFormat() + + def setUp(self): + self.maxDiff = None + + def run_out_test(self, fname, symbols, expected): + """ + Generate output using out_style, + """ + result, expected = gen_output(fname, self.out_style, + symbols, expected) + + self.assertEqual(result, expected) + +class KdocItemToRest(unittest.TestCase): + out_style = RestFormat() + + def setUp(self): + self.maxDiff = None + + def run_out_test(self, fname, symbols, expected): + """ + Generate output using out_style, + """ + result, expected = gen_output(fname, self.out_style, symbols, + expected, relax_whitespace=True) + + self.assertEqual(result, expected) + + +class CToMan(unittest.TestCase): + out_style = ManFormat() + config = MockKdocConfig() + xforms = CTransforms() + + def setUp(self): + self.maxDiff = None + + def run_out_test(self, fname, source, expected): + """ + Generate output using out_style, + """ + patcher = patch('builtins.open', + new_callable=mock_open, read_data=source) + + kernel_doc = KernelDoc(self.config, fname, self.xforms) + + with patcher: + export_table, entries = kernel_doc.parse_kdoc() + + result, expected = gen_output(fname, self.out_style, + entries, expected, config=self.config) + + self.assertEqual(result, expected) + + +class CToRest(unittest.TestCase): + out_style = RestFormat() + config = MockKdocConfig() + xforms = CTransforms() + + def setUp(self): + self.maxDiff = None + + def run_out_test(self, fname, source, expected): + """ + Generate output using out_style, + """ + patcher = patch('builtins.open', + new_callable=mock_open, read_data=source) + + kernel_doc = KernelDoc(self.config, fname, self.xforms) + + with patcher: + export_table, entries = kernel_doc.parse_kdoc() + + result, expected = gen_output(fname, self.out_style, entries, + expected, relax_whitespace=True, + config=self.config) + + self.assertEqual(result, expected) + + +# +# Selftest class +# +class TestSelfValidate(GenerateKdocItem): + """ + Tests to check if logic inside GenerateKdocItem.run_test() is working. + """ + + SOURCE = """ + /** + * function3: Exported function + * @arg1: @arg1 does nothing + * + * Does nothing + * + * return: + * always return 0. + */ + int function3(char *arg1) { return 0; }; + EXPORT_SYMBOL(function3); + """ + + EXPECTED = [{ + 'name': 'function3', + 'type': 'function', + 'declaration_start_line': 2, + + 'sections_start_lines': { + 'Description': 4, + 'Return': 7, + }, + 'sections': { + 'Description': 'Does nothing\n\n', + 'Return': '\nalways return 0.\n' + }, + + 'sections_start_lines': { + 'Description': 4, + 'Return': 7, + }, + + 'parameterdescs': {'arg1': '@arg1 does nothing\n'}, + 'parameterlist': ['arg1'], + 'parameterdesc_start_lines': {'arg1': 3}, + 'parametertypes': {'arg1': 'char *arg1'}, + + 'other_stuff': { + 'func_macro': False, + 'functiontype': 'int', + 'purpose': 'Exported function', + 'typedef': False + }, + }] + + EXPORTS = {"function3"} + + def test_parse_pass(self): + """ + Test if export_symbol is properly handled. + """ + self.run_test(self.SOURCE, self.EXPECTED, self.EXPORTS) + + @unittest.expectedFailure + def test_no_exports(self): + """ + Test if export_symbol is properly handled. + """ + self.run_test(self.SOURCE, [], {}) + + @unittest.expectedFailure + def test_with_empty_expected(self): + """ + Test if export_symbol is properly handled. + """ + self.run_test(self.SOURCE, [], self.EXPORTS) + + @unittest.expectedFailure + def test_with_unfilled_expected(self): + """ + Test if export_symbol is properly handled. + """ + self.run_test(self.SOURCE, [{}], self.EXPORTS) + + @unittest.expectedFailure + def test_with_default_expected(self): + """ + Test if export_symbol is properly handled. + """ + self.run_test(self.SOURCE, [self.DEFAULT.copy()], self.EXPORTS) + +# +# Class and logic to create dynamic tests from YAML +# + +class KernelDocDynamicTests(): + """ + Dynamically create a set of tests from a YAML file. + """ + + @classmethod + def create_parser_test(cls, name, fname, source, symbols, exports): + """ + Return a function that will be attached to the test class. + """ + def test_method(self): + """Lambda-like function to run tests with provided vars""" + self.run_parser_test(source, symbols, exports, fname) + + test_method.__name__ = f"test_gen_{name}" + + setattr(CToKdocItem, test_method.__name__, test_method) + + @classmethod + def create_out_test(cls, name, fname, symbols, out_type, data): + """ + Return a function that will be attached to the test class. + """ + def test_method(self): + """Lambda-like function to run tests with provided vars""" + self.run_out_test(fname, symbols, data) + + test_method.__name__ = f"test_{out_type}_{name}" + + if out_type == "man": + setattr(KdocItemToMan, test_method.__name__, test_method) + else: + setattr(KdocItemToRest, test_method.__name__, test_method) + + @classmethod + def create_src2out_test(cls, name, fname, source, out_type, data): + """ + Return a function that will be attached to the test class. + """ + def test_method(self): + """Lambda-like function to run tests with provided vars""" + self.run_out_test(fname, source, data) + + test_method.__name__ = f"test_{out_type}_{name}" + + if out_type == "man": + setattr(CToMan, test_method.__name__, test_method) + else: + setattr(CToRest, test_method.__name__, test_method) + + @classmethod + def create_tests(cls): + """ + Iterate over all scenarios and add a method to the class for each. + + The logic in this function assumes a valid test that are compliant + with kdoc-test-schema.yaml. There is an unit test to check that. + As such, it picks mandatory values directly, and uses get() for the + optional ones. + """ + + with open(TEST_FILE, encoding="utf-8") as fp: + testset = yaml.safe_load(fp) + + tests = testset["tests"] + + for idx, test in enumerate(tests): + name = test["name"] + fname = test["fname"] + source = test["source"] + expected_list = test["expected"] + + exports = test.get("exports", []) + + # + # The logic below allows setting up to 5 types of test: + # 1. from source to kdoc_item: test KernelDoc class; + # 2. from kdoc_item to man: test ManOutput class; + # 3. from kdoc_item to rst: test RestOutput class; + # 4. from source to man without checking expected KdocItem; + # 5. from source to rst without checking expected KdocItem. + # + for expected in expected_list: + kdoc_item = expected.get("kdoc_item") + man = expected.get("man", []) + rst = expected.get("rst", []) + + if kdoc_item: + if isinstance(kdoc_item, dict): + kdoc_item = [kdoc_item] + + symbols = [] + + for arg in kdoc_item: + arg["fname"] = fname + arg["start_line"] = 1 + + symbols.append(KdocItem.from_dict(arg)) + + if source: + cls.create_parser_test(name, fname, source, + symbols, exports) + + if man: + cls.create_out_test(name, fname, symbols, "man", man) + + if rst: + cls.create_out_test(name, fname, symbols, "rst", rst) + + elif source: + if man: + cls.create_src2out_test(name, fname, source, "man", man) + + if rst: + cls.create_src2out_test(name, fname, source, "rst", rst) + +KernelDocDynamicTests.create_tests() + +# +# Run all tests +# +if __name__ == "__main__": + run_unittest(__file__) diff --git a/tools/unittests/test_kdoc_test_schema.py b/tools/unittests/test_kdoc_test_schema.py new file mode 100755 index 000000000000..9eceeba00440 --- /dev/null +++ b/tools/unittests/test_kdoc_test_schema.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +Unit‑test driver for kernel‑doc YAML tests. + +Two kinds of tests are defined: + +* **Schema‑validation tests** – if ``jsonschema`` is available, the + YAML files in this directory are validated against the JSON‑Schema + described in ``kdoc-test-schema.yaml``. When the library is not + present, a warning is emitted and the validation step is simply + skipped – the dynamic kernel‑doc tests still run. + +* **Kernel‑doc tests** – dynamically generate one test method per + scenario in ``kdoc-test.yaml``. Each method simply forwards + the data to ``self.run_test`` – you only need to implement that + helper in your own code. + +File names are kept as module‑level constants so that the +implementation stays completely independent of ``pathlib``. +""" + +import os +import sys +import warnings +import yaml +import unittest +from typing import Any, Dict, List + +SRC_DIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(SRC_DIR, "../lib/python")) + +from unittest_helper import run_unittest + + +# +# Files to read +# +BASE = os.path.realpath(os.path.dirname(__file__)) + +SCHEMA_FILE = os.path.join(BASE, "kdoc-test-schema.yaml") +TEST_FILE = os.path.join(BASE, "kdoc-test.yaml") + +# +# Schema‑validation test +# +class TestYAMLSchemaValidation(unittest.TestCase): + """ + Checks if TEST_FILE matches SCHEMA_FILE. + """ + + @classmethod + def setUpClass(cls): + """ + Import jsonschema if available. + """ + + try: + from jsonschema import Draft7Validator + except ImportError: + print("Warning: jsonschema package not available. Skipping schema validation") + cls.validator = None + return + + with open(SCHEMA_FILE, encoding="utf-8") as fp: + cls.schema = yaml.safe_load(fp) + + cls.validator = Draft7Validator(cls.schema) + + def test_kdoc_test_yaml_followsschema(self): + """ + Run jsonschema validation if the validator is available. + If not, emit a warning and return without failing. + """ + if self.validator is None: + return + + with open(TEST_FILE, encoding="utf-8") as fp: + data = yaml.safe_load(fp) + + errors = self.validator.iter_errors(data) + + msgs = [] + for error in errors: + msgs.append(error.message) + + if msgs: + self.fail("Schema validation failed:\n\t" + "\n\t".join(msgs)) + +# -------------------------------------------------------------------- +# Entry point +# -------------------------------------------------------------------- +if __name__ == "__main__": + run_unittest(__file__) diff --git a/tools/unittests/test_tokenizer.py b/tools/unittests/test_tokenizer.py index 5634b4a7283e..d1f3c565b9cf 100755 --- a/tools/unittests/test_tokenizer.py +++ b/tools/unittests/test_tokenizer.py @@ -46,10 +46,17 @@ def make_tokenizer_test(name, data): # # Check if logger is working # - if "log_level" in data: - with self.assertLogs('kdoc.c_lex', level='ERROR') as cm: + if "log_msg" in data: + with self.assertLogs() as cm: tokenizer = CTokenizer(data["source"]) + msg_found = False + for result in cm.output: + if data["log_msg"] in result: + msg_found = True + + self.assertTrue(msg_found, f"Missing log {data['log_msg']}") + return # @@ -124,7 +131,7 @@ TESTS_TOKENIZER = { "mismatch_error": { "source": "int a$ = 5;", # $ is illegal - "log_level": "ERROR", + "log_msg": "Unexpected token", }, }