fix namegen for anon/unnamed

atom.mk

@@ -17,6 +17,7 @@ LOCAL_COPY_FILES := \
 	ctypeslib/codegen/util.py:usr/lib/python/site-packages/ctypeslib/codegen/ \
 	ctypeslib/data/fundamental_type_name.tpl:usr/lib/python/site-packages/ctypeslib/data/ \
 	ctypeslib/data/headers.tpl:usr/lib/python/site-packages/ctypeslib/data/ \
+	ctypeslib/data/ioctl.tpl:usr/lib/python/site-packages/ctypeslib/data/ \
 	ctypeslib/data/pointer_type.tpl:usr/lib/python/site-packages/ctypeslib/data/ \
 	ctypeslib/data/string_cast.tpl:usr/lib/python/site-packages/ctypeslib/data/ \
 	ctypeslib/data/structure_type.tpl:usr/lib/python/site-packages/ctypeslib/data/ \

ctypeslib/codegen/clangparser.py

@@ -11,6 +11,7 @@
 from ctypeslib.codegen import typedesc
 from ctypeslib.codegen import typehandler
 from ctypeslib.codegen import util
+from ctypeslib.oset import OSet
 from ctypeslib.codegen.handler import DuplicateDefinitionException
 from ctypeslib.codegen.handler import InvalidDefinitionError
 from ctypeslib.codegen.handler import InvalidTranslationUnitException
@@ -70,7 +71,7 @@ class Clang_Parser:
     def __init__(self, flags):
         self.all = collections.OrderedDict()
         # a shortcut to identify registered decl in cases of records
-        self.all_set = set()
+        self.all_set = OSet()
         self.cpp_data = {}
         self._unhandled = []
         self.fields = {}
@@ -83,7 +84,7 @@ def __init__(self, flags):
         self.cursorkind_handler = cursorhandler.CursorHandler(self)
         self.typekind_handler = typehandler.TypeHandler(self)
         self.__filter_location = None
-        self.__processed_location = set()
+        self.__processed_location = OSet()
 
     def init_parsing_options(self):
         """Set the Translation Unit to skip functions bodies per default."""
@@ -419,5 +420,41 @@ def get_result(self):
             if isinstance(i, interesting):
                 result.append(i)
 
-        log.debug("parsed items order: %s", result)
-        return result
+        # toposort them
+        name2i = {}
+        for i,r in enumerate(result):
+            name2i[r.name] = i
+            if isinstance(r, typedesc.Enumeration):
+                for v in r.values:
+                    name2i[v.name] = i
+
+        edges = [list() for i in range(len(result))]
+        for i,r in enumerate(result):
+            x = util.all_undefined_identifier(r)
+            for x in util.all_undefined_identifier(r):
+                if x.name in name2i:
+                    edges[name2i[x.name]].append(i)
+
+        inDegree = [0] * len(result)
+        for i in range(len(result)):
+            for j in edges[i]:
+                inDegree[j] += 1
+
+        frontier = []
+        for i in range(len(result)):
+            if inDegree[i] == 0:
+                frontier.append(i)
+
+        order = []
+        while frontier:
+            i = min(frontier)
+            order.append(i)
+            frontier.remove(i)
+            for j in edges[i]:
+                inDegree[j] -= 1
+                if inDegree[j] == 0:
+                    frontier.append(j)
+
+        final_result = [result[i] for i in order]
+        log.debug("parsed items order: %s", final_result)
+        return final_result

ctypeslib/codegen/codegenerator.py

@@ -22,6 +22,7 @@
 from ctypeslib.codegen import config
 from ctypeslib.codegen import typedesc
 from ctypeslib.codegen import util
+from ctypeslib.oset import OSet
 from ctypeslib.library import Library
 
 log = logging.getLogger("codegen")
@@ -50,8 +51,8 @@ def __init__(self, output, cfg):
         self.macros = 0
         self.cross_arch_code_generation = cfg.cross_arch
         # what record dependency were generated
-        self.head_generated = set()
-        self.body_generated = set()
+        self.head_generated = OSet()
+        self.body_generated = OSet()
 
     # pylint: disable=method-hidden
     def enable_fundamental_type_wrappers(self):
@@ -95,6 +96,12 @@ def enable_structure_type(self):
         headers = pkgutil.get_data("ctypeslib", "data/structure_type.tpl").decode()
         print(headers, file=self.imports)
 
+    def enable_ioctl(self):
+        self.enable_ioctl = lambda: True
+        headers = pkgutil.get_data("ctypeslib", "data/ioctl.tpl").decode()
+        self.names += ["_IO", "_IOR", "_IOW", "_IOWR"]
+        print(headers, file=self.imports)
+
     def enable_string_cast(self):
         """
         If a structure type is used, declare our ctypes.Structure extension type
@@ -203,15 +210,28 @@ def Macro(self, macro):
         # 2. or get a flag in macro that tells us if something contains undefinedIdentifier
         # is not code-generable ?
         # codegen should decide what codegen can do.
+        unknowns = util.all_undefined_identifier(macro)
+        for x in unknowns:
+            if x.name in {"_IO", "_IOW", "_IOR", "_IOWR"}: self.enable_ioctl()
+
+        all_known = all(x.name in self.names for x in unknowns)
+        if isinstance(macro.body, list) or isinstance(macro.body, str):
+            for b in ['?', ':']: all_known &= b not in macro.body
         if macro.args:
-            print("# def %s%s:  # macro" % (macro.name, macro.args), file=self.stream)
-            print("#    return %s  " % macro.body, file=self.stream)
-        elif util.contains_undefined_identifier(macro):
+            if all_known:
+                print("def %s%s:  # macro" % (macro.name, macro.args), file=self.stream)
+                print("   return %s  " % macro.body, file=self.stream)
+            else:
+                print("# def %s%s:  # macro" % (macro.name, macro.args), file=self.stream)
+                print("#    return %s  " % macro.body, file=self.stream)
+        elif not all_known:
             # we can't handle that, we comment it out
-            if isinstance(macro.body, typedesc.UndefinedIdentifier):
-                print("# %s = %s # macro" % (macro.name, macro.body.name), file=self.stream)
-            else:  # we assume it's a list
+            if isinstance(macro.body, list):
                 print("# %s = %s # macro" % (macro.name, " ".join([str(_) for _ in macro.body])), file=self.stream)
+            else:
+                print("# %s = %s # macro" % (macro.name, macro.body), file=self.stream)
+        elif isinstance(macro.body, list):
+            print("%s = %s # macro (from list)" % (macro.name, " ".join([str(_) for _ in macro.body])), file=self.stream)
         elif isinstance(macro.body, bool):
             print("%s = %s # macro" % (macro.name, macro.body), file=self.stream)
             self.macros += 1
@@ -551,7 +571,7 @@ def Structure(self, struct):
             return
         self.enable_structure_type()
         self._structures += 1
-        depends = set()
+        depends = OSet()
         # We only print a empty struct.
         if struct.members is None:
             log.info("No members for: %s", struct.name)

ctypeslib/codegen/cursorhandler.py

@@ -481,9 +481,11 @@ def _literal_handling(self, cursor):
             value = self._clean_string_literal(cursor, value)
             return value
         final_value = []
+        skip_till = None
         # code.interact(local=locals())
         log.debug('cursor.type:%s', cursor.type.kind.name)
         for i, token in enumerate(tokens):
+            if skip_till is not None and i <= skip_till: continue
             value = token.spelling
             log.debug('token:%s tk.kd:%11s tk.cursor.kd:%15s cursor.kd:%15s',
                       token.spelling, token.kind.name, token.cursor.kind.name,
@@ -544,25 +546,41 @@ def _literal_handling(self, cursor):
                 elif token.kind == TokenKind.IDENTIFIER:  # noqa
                     # log.debug("Ignored MACRO_DEFINITION token identifier : %s", value)
                     # Identifier in Macro... Not sure what to do with that.
+                    if i > 0 and tokens[i-1].spelling == "struct":
+                        final_value.pop()
+                        value = "struct_" + value
+
                     if self.is_registered(value):
                         # FIXME: if Macro is not a simple value replace, it should not be registered in the first place
                         # parse that, try to see if there is another Macro in there.
-                        value = self.get_registered(value).body
+                        if isinstance(self.get_registered(value), typedesc.Typedef):
+                            value = None
+                        elif isinstance(self.get_registered(value), typedesc.Macro):
+                            if self.get_registered(value).args is None: value = self.get_registered(value).toknes
+                        else:
+                            value = self.get_registered(value).body
                         log.debug("Found MACRO_DEFINITION token identifier : %s", value)
                     else:
                         value = typedesc.UndefinedIdentifier(value)
                         log.debug("Undefined MACRO_DEFINITION token identifier : %s", value)
                     pass
                 elif token.kind == TokenKind.KEYWORD:  # noqa
                     log.debug("Got a MACRO_DEFINITION referencing a KEYWORD token.kind: %s", token.kind.name)
-                    value = typedesc.UndefinedIdentifier(value)
+                    replacer = {"void": "void", "struct": "struct", "sizeof": "ctypes.sizeof"}
+                    value=replacer.get(value, None)
+                    # value = typedesc.UndefinedIdentifier(value)
                 elif token.kind in [TokenKind.COMMENT, TokenKind.PUNCTUATION]:  # noqa
                     # log.debug("Ignored MACRO_DEFINITION token.kind: %s", token.kind.name)
-                    pass
+                    replacer = {"||": " or ", "&&": " and "}
+                    value = replacer.get(value, value)
+                    if final_value[-1] == "(" and value == ")":
+                        final_value.pop()
+                        value = None
 
             # add token
             if value is not None:
-                final_value.append(value)
+                if isinstance(value, list): final_value += value
+                else: final_value.append(value)
         # return the EXPR
         # code.interact(local=locals())
         # FIXME, that will break. We need constant type return
@@ -1112,9 +1130,10 @@ def MACRO_DEFINITION(self, cursor):
         value = True
         # args should be filled when () are in tokens,
         args = None
+        unknowns = None
         if isinstance(tokens, list):
             # TODO, if there is an UndefinedIdentifier, we need to scrap the whole thing to comments.
-            # unknowns = [_ for _ in tokens if isinstance(_, typedesc.UndefinedIdentifier)]
+            unknowns = [_ for _ in tokens if isinstance(_, typedesc.UndefinedIdentifier)]
             # if len(unknowns) > 0:
             #     value = tokens
             # elif len(tokens) == 2:
@@ -1126,21 +1145,52 @@ def MACRO_DEFINITION(self, cursor):
             elif tokens[1] == '(':
                 # #107, differentiate between function-like macro and expression in ()
                 # valid tokens for us are are '()[0-9],.e' and terminating LluU
-                if any(filter(lambda x: isinstance(x, typedesc.UndefinedIdentifier), tokens)):
+                args_tokens = [str(_) for _ in tokens[1:tokens.index(')')+1]]
+                body_tokens = [str(_) for _ in tokens[tokens.index(')')+1:]]
+
+                bal = 0
+                body_tokens_good = len(body_tokens) > 0
+                for t in body_tokens:
+                    if t == '(': bal += 1
+                    if t == ')': bal -= 1
+                    if bal < 0: body_tokens_good = False
+                body_tokens_good &= (bal == 0)
+
+                if any(filter(lambda x: isinstance(x, typedesc.UndefinedIdentifier), tokens)) and body_tokens_good:
                     # function macro or an expression.
-                    str_tokens = [str(_) for _ in tokens[1:tokens.index(')')+1]]
-                    args = ''.join(str_tokens).replace(',', ', ')
-                    str_tokens = [str(_) for _ in tokens[tokens.index(')')+1:]]
-                    value = ''.join(str_tokens)
+                    for x in tokens[1:tokens.index(')')+1]:
+                        if not isinstance(x, typedesc.UndefinedIdentifier): continue
+                        rm, found = [], 0
+                        for i in range(len(unknowns)):
+                            if unknowns[i].name == x.name:
+                                rm.append(i)
+                                found += 1
+                        if found < 2:
+                            log.debug(f'MACRO: skipping gen of {tokens}')
+                            unknowns.append(typedesc.UndefinedIdentifier("mark_as_broken"))
+
+                        for x in rm[::-1]: unknowns.pop(x)
+
+                    args = ''.join(args_tokens).replace(',', ', ')
+                    value = ''.join(body_tokens)
+                elif not body_tokens_good:
+                    value = ''.join((str(_) for _ in tokens[1:]))
+                    if value.find("u64") >= 0: unknowns.append(typedesc.UndefinedIdentifier("mark_as_broken"))
+                    if value.find("u32") >= 0: unknowns.append(typedesc.UndefinedIdentifier("mark_as_broken"))
+                    if value.find("i64") >= 0: unknowns.append(typedesc.UndefinedIdentifier("mark_as_broken"))
+                    if value.find("i32") >= 0: unknowns.append(typedesc.UndefinedIdentifier("mark_as_broken"))
+                    if value.find("void*") >= 0: unknowns.append(typedesc.UndefinedIdentifier("mark_as_broken"))
+                    if value.find("sizeof") >= 0: unknowns.append(typedesc.UndefinedIdentifier("mark_as_broken"))
                 else:
                     value = ''.join((str(_) for _ in tokens[1:tokens.index(')') + 1]))
-            elif len(tokens) > 2:
+            elif len(tokens) > 2 and len(unknowns) == 0 and ':' not in tokens:
                 # #define key a b c
-                value = list(tokens[1:])
+                # value = list(tokens[1:])
+                value = ' '.join(tokens[1:])
             else:
                 # FIXME no reach ?!
                 # just merge the list of tokens
-                value = ' '.join(tokens[1:])
+                value = list(tokens[1:])
         elif isinstance(tokens, str):
             # #define only
             value = True
@@ -1154,11 +1204,11 @@ def MACRO_DEFINITION(self, cursor):
         if name in ['NULL', '__thread'] or value in ['__null', '__thread']:
             value = None
         log.debug('MACRO: #define %s%s %s', name, args or '', value)
-        obj = typedesc.Macro(name, args, value)
+        obj = typedesc.Macro(name, args, value, tokens[1:], unknowns)
         try:
             self.register(name, obj)
         except DuplicateDefinitionException:
-            log.info('Redefinition of %s %s->%s', name, self.parser.all[name].args, value)
+            log.info('Redefinition of %s %s->%s', name, getattr(self.parser.all[name], 'args', ''), value)
             # HACK
             self.parser.all[name] = obj
         self.set_location(obj, cursor)

ctypeslib/codegen/handler.py

@@ -5,6 +5,7 @@
 from ctypeslib.codegen import typedesc
 from ctypeslib.codegen.util import log_entity
 
+import re
 import logging
 log = logging.getLogger('handler')
 
@@ -149,6 +150,13 @@ def get_unique_name(self, cursor, field_name=None):
                 #import code
                 #code.interact(local=locals())
                 return ''
+
+        # rewrite with unnamed/anonymous parts.
+        pattern = r'\s+\(unnamed at .*/(.*?).h:(\d+):\d+\)'
+        name = re.sub(pattern, lambda x: f"_{x.group(1)}_h_{x.group(2)}", name)
+        pattern = r'\s+\(anonymous at .*/(.*?).h:(\d+):\d+\)'
+        name = re.sub(pattern, lambda x: f"_{x.group(1)}_h_{x.group(2)}", name)
+
         if cursor.kind in [CursorKind.STRUCT_DECL,CursorKind.UNION_DECL,
                                  CursorKind.CLASS_DECL, CursorKind.CXX_BASE_SPECIFIER]:
             names= {CursorKind.STRUCT_DECL: 'struct',

ctypeslib/codegen/typedesc.py

@@ -70,13 +70,15 @@ class Macro(T):
 
     """a C preprocessor definition with arguments"""
 
-    def __init__(self, name, args, body):
+    def __init__(self, name, args, body, toknes, unknowns=None):
         """all arguments are strings, args is the literal argument list
         *with* the parens around it:
         Example: Macro("CD_INDRIVE", "(status)", "((int)status > 0)")"""
         self.name = name
         self.args = args
         self.body = body
+        self.toknes = toknes
+        self.unknowns = unknowns
 
 
 class File(T):

ctypeslib/codegen/util.py

@@ -170,6 +170,30 @@ def _list_contains_undefined_identifier(l):
 
     return False
 
+def all_undefined_identifier(macro):
+    rres = []
+    if isinstance(macro, typedesc.Macro) and macro.unknowns is not None: rres += macro.unknowns
+    if not hasattr(macro, 'body'): return rres
+
+    # body is undefined
+    if isinstance(macro.body, typedesc.UndefinedIdentifier):
+        return rres + [macro.body]
+
+    def _list_contains_undefined_identifier(l):
+        res = []
+        for b in l:
+            if isinstance(b, typedesc.UndefinedIdentifier):
+                res += [b]
+            if isinstance(b, list):
+                res += all_undefined_identifier(b)
+        return res
+
+    # or one item is undefined
+    if isinstance(macro.body, list):
+        return _list_contains_undefined_identifier(macro.body)
+
+    return rres
+
 
 def token_is_string(token):
     # we need at list 2 delimiters in there

ctypeslib/data/ioctl.tpl

@@ -0,0 +1,12 @@
+
+import fcntl, functools
+
+def _do_ioctl(__idir, __base, __nr, __user_struct, __fd, **kwargs):
+  ret = fcntl.ioctl(__fd, (__idir<<30) | (ctypes.sizeof(made := __user_struct(**kwargs))<<16) | (__base<<8) | __nr, made)
+  if ret != 0: raise RuntimeError(f"ioctl returned {ret}")
+  return made
+
+def _IO(base, nr): return functools.partial(_do_ioctl, 0, ord(base) if isinstance(base, str) else base, nr, None)
+def _IOW(base, nr, type): return functools.partial(_do_ioctl, 1, ord(base) if isinstance(base, str) else base, nr, type)
+def _IOR(base, nr, type): return functools.partial(_do_ioctl, 2, ord(base) if isinstance(base, str) else base, nr, type)
+def _IOWR(base, nr, type): return functools.partial(_do_ioctl, 3, ord(base) if isinstance(base, str) else base, nr, type)

ctypeslib/data/structure_type.tpl

@@ -1,4 +1,6 @@
 class AsDictMixin:
+    import sys
+    if sys.version_info >= (3, 14): _layout_ = 'ms'
     @classmethod
     def as_dict(cls, self):
         result = {}