python实现词法分析器
作者:追_梦_者 时间:2022-09-12 15:03:18
简单Python词法分析器实现,供大家参考,具体内容如下
词法分析器状态转换图:
词法分析器总流程图:
预处理程序:
词法分析器:
词法分析器程序详细设计
详细代码实现:
#!/usr/bin/env python3.4
# coding=utf-8
import sys
import string
keywards = {}
# 关键字部分
keywards['False'] = 101
keywards['class'] = 102
keywards['finally'] = 103
keywards['is'] = 104
keywards['return'] = 105
keywards['None'] = 106
keywards['continue'] = 107
keywards['for'] = 108
keywards['lambda'] = 109
keywards['try'] = 110
keywards['True'] = 111
keywards['def'] = 112
keywards['from'] = 113
keywards['nonlocal'] = 114
keywards['while'] = 115
keywards['and'] = 116
keywards['del'] = 117
keywards['global'] = 118
keywards['not'] = 119
keywards['with'] = 120
keywards['as'] = 121
keywards['elif'] = 122
keywards['if'] = 123
keywards['or'] = 124
keywards['yield'] = 125
keywards['assert'] = 126
keywards['else'] = 127
keywards['import'] = 128
keywards['pass'] = 129
keywards['break'] = 130
keywards['except'] = 131
keywards['in'] = 132
keywards['raise'] = 133
# 符号
keywards['+'] = 201
keywards['-'] = 202
keywards['*'] = 203
keywards['/'] = 204
keywards['='] = 205
keywards[':'] = 206
keywards['<'] = 207
keywards['>'] = 208
keywards['%'] = 209
keywards['&'] = 210
keywards['!'] = 211
keywards['('] = 212
keywards[')'] = 213
keywards['['] = 214
keywards[']'] = 215
keywards['{'] = 216
keywards['}'] = 217
keywards['#'] = 218
keywards['|'] = 219
keywards[','] = 220
# 变量
# keywards['var'] = 301
# 常量
# keywards['const'] = 401
# Error
# keywards['const'] = 501
signlist = {}
# 预处理函数,将文件中的空格,换行等无关字符处理掉
def pretreatment(file_name):
try:
fp_read = open(file_name, 'r')
fp_write = open('file.tmp', 'w')
sign = 0
while True:
read = fp_read.readline()
if not read:
break
length = len(read)
i = -1
while i < length - 1:
i += 1
if sign == 0:
if read[i] == ' ':
continue
if read[i] == '#':
break
elif read[i] == ' ':
if sign == 1:
continue
else:
sign = 1
fp_write.write(' ')
elif read[i] == '\t':
if sign == 1:
continue
else:
sign = 1
fp_write.write(' ')
elif read[i] == '\n':
if sign == 1:
continue
else:
fp_write.write(' ')
sign = 1
elif read[i] == '"':
fp_write.write(read[i])
i += 1
while i < length and read[i] != '"':
fp_write.write(read[i])
i += 1
if i >= length:
break
fp_write.write(read[i])
elif read[i] == "'":
fp_write.write(read[i])
i += 1
while i < length and read[i] != "'":
fp_write.write(read[i])
i += 1
if i >= length:
break
fp_write.write(read[i])
else:
sign = 3
fp_write.write(read[i])
except Exception:
print(file_name, ': This FileName Not Found!')
def save(string):
if string in keywards.keys():
if string not in signlist.keys():
signlist[string] = keywards[string]
else:
try:
float(string)
save_const(string)
except ValueError:
save_var(string)
def save_var(string):
if string not in signlist.keys():
if len(string.strip()) < 1:
pass
else:
if is_signal(string) == 1:
signlist[string] = 301
else:
signlist[string] = 501
def save_const(string):
if string not in signlist.keys():
signlist[string] = 401
def save_error(string):
if string not in signlist.keys():
signlist[string] = 501
def is_signal(s):
if s[0] == '_' or s[0] in string.ascii_letters:
for i in s:
if i in string.ascii_letters or i == '_' or i in string.digits:
pass
else:
return 0
return 1
else:
return 0
def recognition(filename):
try:
fp_read = open(filename, 'r')
string = ""
sign = 0
while True:
read = fp_read.read(1)
if not read:
break
if read == ' ':
if len(string.strip()) < 1:
sign = 0
pass
else:
if sign == 1 or sign == 2:
string += read
else:
save(string)
string = ""
sign = 0
elif read == '(':
if sign == 1 or sign == 2:
string += read
else:
save(string)
string = ""
save('(')
elif read == ')':
if sign == 1 or sign == 2:
string += read
else:
save(string)
string = ""
save(')')
elif read == '[':
if sign == 1 or sign == 2:
string += read
else:
save(string)
string = ""
save('[')
elif read == ']':
if sign == 1 or sign == 2:
string += read
else:
save(string)
string = ""
save(']')
elif read == '{':
if sign == 1 or sign == 2:
string += read
else:
save(string)
string = ""
save('{')
elif read == '}':
if sign == 1 or sign == 2:
string += read
else:
save(string)
string = ""
save('}')
elif read == '<':
save(string)
string = ""
save('<')
elif read == '>':
save(string)
string = ""
save('>')
elif read == ',':
save(string)
string = ""
save(',')
elif read == "'":
string += read
if sign == 1:
sign = 0
save_const(string)
string = ""
else:
if sign != 2:
sign = 1
elif read == '"':
string += read
if sign == 2:
sign = 0
save_const(string)
string = ""
else:
if sign != 1:
sign = 2
elif read == ':':
if sign == 1 or sign == 2:
string += read
else:
save(string)
string = ""
save(':')
elif read == '+':
save(string)
string = ""
save('+')
elif read == '=':
save(string)
string = ""
save('=')
else:
string += read
except Exception as e:
print(e)
def main():
if len(sys.argv) < 2:
print("Please Input FileName")
else:
pretreatment(sys.argv[1])
recognition('file.tmp')
for i in signlist.keys():
print("(", signlist[i], ",", i, ")")
if __name__ == '__main__':
main()
来源:https://blog.csdn.net/IT_DREAM_ER/article/details/53485583
标签:python,词法分析器
0
投稿
猜你喜欢
SQL Server 2005实现数据库缓存依赖
2009-05-07 13:20:00
selenium+python实现自动登录脚本
2021-09-30 01:36:19
Go语言atomic.Value如何不加锁保证数据线程安全?
2024-04-25 13:16:52
python excel转换csv代码实例
2023-10-30 15:19:53
Python容器使用的5个技巧和2个误区总结
2023-04-09 04:37:01
详解使用Vue.Js结合Jquery Ajax加载数据的两种方式
2023-07-02 17:07:39
Python实现邮件的批量发送的示例代码
2023-08-09 07:47:57
C#调用Python模块的方法
2021-04-13 15:29:10
Python调用飞书发送消息的示例
2022-10-20 14:21:23
vscode 远程调试python的方法
2021-09-07 23:00:59
python实现视频压缩功能
2023-12-14 02:48:36
对numpy和pandas中数组的合并和拆分详解
2021-04-23 22:51:50
python实现多进程通信实例分析
2023-04-08 19:44:05
PHP的PDO预定义常量讲解
2023-06-09 13:35:54
Python实现人机中国象棋游戏
2023-01-28 21:33:52
Python定时任务sched模块用法示例
2021-12-17 12:49:42
Oracle生成随机数字、字符串、日期、验证码及 UUID的方法
2024-01-23 08:06:03
Python初学者必须掌握的25个内置函数详解
2022-07-02 16:09:21
解决Windows 7下安装Oracle 11g相关问题的方法
2024-01-22 05:19:19
python2.7安装opencv-python很慢且总是失败问题
2021-01-10 06:11:15