process.py (6837B)
1 import sys 2 import re 3 from dataclasses import dataclass 4 from collections import namedtuple 5 import tokenize 6 7 Macro = namedtuple("Macro", "args func") 8 9 def check_bare(tokens, directive_name): 10 if len(tokens) == 1: 11 print(f"Error: bare {directive_name}", file=sys.stderr) 12 exit(1) 13 14 def handle_define(tokens, state): 15 check_bare(tokens, "#define") 16 if len(tokens) > 2: 17 if (m := re.search('\((.*?)\)', tokens[1])): 18 content = " ".join(tokens[2:]) 19 args = m.group(0)[1:-1].split(",") 20 21 def macro(inps): 22 copy = content 23 for (i, inp) in enumerate(inps): 24 copy = copy.replace(args[i], inp) 25 return copy 26 27 state.defs[tokens[1][:m.span()[0]]] = Macro( 28 len(args), 29 macro 30 ) 31 32 else: 33 state.defs[tokens[1]] = " ".join(tokens[2:]) 34 else: 35 state.defs[tokens[1]] = "" 36 37 def handle_undef(tokens, state): 38 check_bare(tokens, "#undef") 39 try: 40 state.defs.pop(tokens[1]) 41 except: 42 pass 43 44 def handle_ifdef(tokens, state): 45 check_bare(tokens, "#ifdef") 46 if tokens[1] not in state.defs: 47 state.skip = True 48 state.prev_cond = False 49 else: 50 state.prev_cond = True 51 52 def handle_ifndef(tokens, state): 53 check_bare(tokens, "#ifndef") 54 if tokens[1] in state.defs: 55 state.skip = True 56 state.prev_cond = True 57 else: 58 state.prev_cond = False 59 60 def handle_include(tokens, state): 61 check_bare(tokens, "#include") 62 63 if (m := re.search('"(.*?)"', tokens[1])): 64 f = open(m.group(0)[1:-1], "rb") 65 stream = tokenize.tokenize(f.readline) 66 next(stream) 67 for tok in stream: 68 state.out_tokens.append(tok) 69 elif (fname := re.search('<(.*?)>', tokens[1])): 70 print("Error: PYTHON_PATH handling not implemented. Also why would you do this.") 71 exit(1) 72 else: 73 print("Error: malformed #include", file=sys.stderr) 74 exit(1) 75 76 def handle_endif(tokens, state): 77 if state.prev_cond is None: 78 print("Error: #endif without #if", file=sys.stderr) 79 exit(1) 80 if state.skip: 81 state.skip = False 82 state.prev_cond = None 83 84 def handle_else(tokens, state): 85 if state.prev_cond is None: 86 print("Error: #else without #if", file=sys.stderr) 87 exit(1) 88 elif state.prev_cond == True: 89 state.skip=True 90 91 Directive = namedtuple("Directive", "name handler") 92 93 handlers = [ 94 Directive("#define", handle_define), 95 Directive("#ifdef", handle_ifdef), 96 Directive("#endif", handle_endif), 97 Directive("#ifndef", handle_ifndef), 98 Directive("#include", handle_include), 99 Directive("#undef", handle_undef), 100 Directive("#else", handle_else), 101 ] 102 103 def try_handle_directive(tokens, state): 104 if len(tokens) == 0: return False 105 for directive in handlers: 106 if tokens[0] == directive.name: 107 directive.handler(tokens, state) 108 return True 109 return False 110 111 @dataclass 112 class State: 113 defs: dict 114 skip: bool 115 prev_cond: bool 116 out_tokens: list 117 i: int 118 119 def __init__(self, out_tokens): 120 self.defs = { 121 "__COUNTER__": "0", # TODO refactor macro expansion into its own function 122 "__VERSION__": f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}", 123 "__PYTHON__": str(sys.version_info.major), 124 "__PYTHON_MINOR__": str(sys.version_info.minor), 125 "__PYTHON_MICRO__": str(sys.version_info.micro), 126 "__IMPLEMENTATION__": sys.implementation.name, 127 "__FILE_NAME__": __file__, 128 "__BYTE_ORDER__": sys.byteorder, 129 "__ORDER_LITTLE_ENDIAN__": "little", 130 "__ORDER_BIG_ENDIAN__": "big", 131 # TODO __TIMESTAMP__ (last modified of exec'd file) 132 } 133 if sys.flags.optimize: 134 self.defs["__OPTIMIZE__"] = str(sys.flags.optimize) 135 if sys.platform == "linux": 136 self.defs["__linux__"] = "1" 137 if sys.platform == "darwin": 138 self.defs["__APPLE__"] = "1" 139 140 self.skip = False 141 self.prev_cond = None 142 self.out_tokens = out_tokens 143 self.i = 0 144 145 def join_tokens(tokens): 146 if len(tokens) == 0: return "" 147 out = tokens[0].string 148 indent = 0 149 for i in range(1, len(tokens)): 150 if tokens[i].type == tokenize.INDENT: 151 indent += 1 152 continue 153 elif tokens[i].type == tokenize.DEDENT: 154 indent -= 1 155 continue 156 elif ((tokens[i-1].type in (tokenize.NEWLINE, tokenize.DEDENT, tokenize.INDENT)) 157 and not (tokens[i].type in (tokenize.NEWLINE, tokenize.COMMENT))): 158 out += " "*indent 159 if tokens[i-1].type == tokenize.NAME and tokens[i].type == tokenize.NAME: 160 out += " " 161 out += tokens[i].string 162 return out 163 164 def preprocess(code): 165 f = io.BytesIO(code) 166 stream = tokenize.tokenize(f.readline) 167 168 out_tokens = [] 169 state = State(out_tokens) 170 next(stream) 171 macro_call = None 172 for token in stream: 173 if state.skip: continue 174 175 if token.type == tokenize.NAME and token.string in state.defs: 176 if isinstance(state.defs[token.string], Macro): 177 macro_call = state.defs[token.string] 178 lparen = next(stream) 179 inps = [] 180 for i in range(macro_call.args): 181 val = next(stream) 182 if (val.string == ","): val = next(stream) 183 inps.append(val.string) 184 rparen = next(stream) 185 print(inps, rparen) 186 if lparen.string != "(" or rparen.string != ")": 187 print("Invalid macro call!", file=sys.stderr) 188 exit(1) 189 out_tokens.append(tokenize.TokenInfo( 190 type = tokenize.NAME, # HACK 191 string = state.defs[token.string].func(inps), 192 start=None, end=None, line=None 193 )) 194 else: 195 out_tokens.append(tokenize.TokenInfo( 196 type=tokenize.NAME, 197 string=state.defs[token.string], 198 start=None, end=None, line=None 199 )) 200 if token.string == "__COUNTER__": 201 state.defs["__COUNTER__"] = str(int(state.defs["__COUNTER__"]) + 1) 202 elif token.type == tokenize.COMMENT: 203 dir_tokens = token.string.split() 204 try_handle_directive(dir_tokens, state) 205 else: 206 out_tokens.append(token) 207 208 return join_tokens(out_tokens)