c-preprocessor

python library for metaprogramming like it's the 1970s
Log | Files | Refs | README

test.py (1630B)

      1 from cpreprocessor import preprocess
      2 import tokenize
      3 import io
      4 
      5 def clean_token(token):
      6     return tokenize.TokenInfo(
      7         type=token.type,
      8         string=token.string,
      9         start=None, end=None, line=None,
     10     )
     11 
     12 def dedent(tokens):
     13     """Decrease indentation of token list by 1 (in-place)."""
     14     for i in range(len(tokens)):
     15         if tokens[i].type == tokenize.INDENT:
     16             del(tokens[i])
     17             break
     18     for i in range(1,len(tokens)-1):
     19         if tokens[-i].type == tokenize.DEDENT:
     20             del(tokens[-i])
     21             break    
     22 
     23 def relevant_token(token):
     24     return (token.type != tokenize.NEWLINE and
     25             token.type != tokenize.NL and
     26             token.type != tokenize.ENCODING and
     27             token.type != tokenize.ENDMARKER)            
     28 
     29 def lexically_equiv(a, b):
     30     a_stream = tokenize.tokenize(io.BytesIO(bytes(a, "utf8")).readline)
     31     b_stream = tokenize.tokenize(io.BytesIO(bytes(b, "utf8")).readline)
     32 
     33     a_toks = list(map(clean_token, filter(relevant_token, a_stream)))
     34     b_toks = list(map(clean_token, filter(relevant_token, b_stream)))    
     35     dedent(a_toks)
     36     dedent(b_toks)
     37     
     38     print(a_toks)
     39     print(b_toks)
     40         
     41     return a_toks == b_toks
     42 
     43 def test_sanity():
     44     assert(lexically_equiv(
     45         preprocess(b"print(1+1)"), "print(1+1)"
     46     ))
     47 
     48 def test_ifdef():
     49     code = b"""
     50     #define TESTING
     51 
     52     #ifdef TESTING
     53     print(1+1)
     54     #endif
     55     """
     56     
     57     assert(lexically_equiv(preprocess(code), "print(1+1)"))
     58 
     59     code = b"""
     60     #ifdef TESTING
     61     print(1+1)
     62     #endif
     63     """
     64     assert(lexically_equiv(preprocess(code), ""))
     65 
     66     
     67     
     68