Python编程
搜索
分类

python 串行执行和并行执行实例

作者：zhaoyangjian724 时间：2022-07-12 07:32:58　

我就废话不多说了，大家还是直接看代码吧！


#coding=utf-8

import threading

import time

import cx_Oracle

from pprint import pprint

import csv

print time.asctime()

table_name = "dbtest.csv"

f = open(table_name + ".csv", "w")

conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')

cursor = conn.cursor()

def query01():

tname = threading.current_thread()

aa=10

# cursor.execute("SELECT * FROM TEST100 WHERE ID = ％s", [aa])

cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")

cursor.execute(None,{'id':aa})

row = cursor.fetchall()

print row

return row

cursor.close()

def query02():

tname = threading.current_thread()

aa=20

# cursor.execute("SELECT * FROM TEST100 WHERE ID = ％s", [aa])

cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")

cursor.execute(None,{'id':aa})

row = cursor.fetchall()

print row

return row

cursor.close()

def query03():

tname = threading.current_thread()

aa=30

# cursor.execute("SELECT * FROM TEST100 WHERE ID = ％s", [aa])

cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")

cursor.execute(None,{'id':aa})

row = cursor.fetchall()

print row

return row

cursor.close()

def query04():

tname = threading.current_thread()

aa=40

# cursor.execute("SELECT * FROM TEST100 WHERE ID = ％s", [aa])

cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")

cursor.execute(None,{'id':aa})

row = cursor.fetchall()

print row

return row

def query05():

tname = threading.current_thread()

aa=50

# cursor.execute("SELECT * FROM TEST100 WHERE ID = ％s", [aa])

cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")

cursor.execute(None,{'id':aa})

row = cursor.fetchall()

print row

return row

cursor.close()

def query06():

tname = threading.current_thread()

aa=60

# cursor.execute("SELECT * FROM TEST100 WHERE ID = ％s", [aa])

cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")

cursor.execute(None,{'id':aa})

row = cursor.fetchall()

print row

return row

cursor.close()

def query07():

tname = threading.current_thread()

aa=70

# cursor.execute("SELECT * FROM TEST100 WHERE ID = ％s", [aa])

cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")

cursor.execute(None,{'id':aa})

row = cursor.fetchall()

return row

cursor.close()

def query08():

tname = threading.current_thread()

aa=80

# cursor.execute("SELECT * FROM TEST100 WHERE ID = ％s", [aa])

cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")

cursor.execute(None,{'id':aa})

row = cursor.fetchall()

print row

return row

cursor.close()

def query09():

tname = threading.current_thread()

aa=90

# cursor.execute("SELECT * FROM TEST100 WHERE ID = ％s", [aa])

cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")

cursor.execute(None,{'id':aa})

row = cursor.fetchall()

print row

return row

cursor.close()

def query10():

tname = threading.current_thread()

aa=100

# cursor.execute("SELECT * FROM TEST100 WHERE ID = ％s", [aa])

cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")

cursor.execute(None,{'id':aa})

row = cursor.fetchall()

print row

return row

cursor.close()

threads = []

t1 = threading.Thread(target=query01, name='query01')

threads.append(t1)

t2 = threading.Thread(target=query02, name='query02')

threads.append(t2)

t2 = threading.Thread(target=query03, name='query03')

threads.append(t2)

t2 = threading.Thread(target=query04, name='query04')

threads.append(t2)

t2 = threading.Thread(target=query05, name='query05')

threads.append(t2)

t2 = threading.Thread(target=query06, name='query06')

threads.append(t2)

t2 = threading.Thread(target=query07, name='query07')

threads.append(t2)

t2 = threading.Thread(target=query08, name='query08')

threads.append(t2)

t2 = threading.Thread(target=query09, name='query09')

threads.append(t2)

t2 = threading.Thread(target=query10, name='query10')

threads.append(t2)

if __name__ == '__main__':

for t in threads:

# t.setDaemon(True)

t.start()

# t.run()

# t.start()

# print '3333333'

print threading.current_thread()

# print t.is_alive()

# print '3333333'

t.join()

print "all over "

print time.asctime()

C:\Python27\python.exe C:/Users/tlcb/PycharmProjects/untitled/a2.py

Wed Mar 28 11:08:19 2018

<_MainThread(MainThread, started 18744)>

[(10, '10boobook10', '10aaaaaaaaaaaa10', '10bbbbbbbbbbbbbbbbb10'), (10, '10sssssssss10', 'tlcb', 'tlcb'), (10, '10boobook10', '10aaaaaaaaaaaa10', '10bbbbbbbbbbbbbbbbb10')]

<_MainThread(MainThread, started 18744)>

[(20, '20boobook20', '20aaaaaaaaaaaa20', '20bbbbbbbbbbbbbbbbb20'), (20, '20boobook20', '20aaaaaaaaaaaa20', '20bbbbbbbbbbbbbbbbb20'), (20, '20boobook20', '20aaaaaaaaaaaa20', '20bbbbbbbbbbbbbbbbb20')]

<_MainThread(MainThread, started 18744)>

[(30, '30boobook30', '30aaaaaaaaaaaa30', '30bbbbbbbbbbbbbbbbb30'), (30, '30boobook30', '30aaaaaaaaaaaa30', '30bbbbbbbbbbbbbbbbb30'), (30, '30boobook30', '30aaaaaaaaaaaa30', '30bbbbbbbbbbbbbbbbb30')]

<_MainThread(MainThread, started 18744)>

[(40, '40boobook40', '40aaaaaaaaaaaa40', '40bbbbbbbbbbbbbbbbb40'), (40, '40boobook40', '40aaaaaaaaaaaa40', '40bbbbbbbbbbbbbbbbb40'), (40, '40boobook40', '40aaaaaaaaaaaa40', '40bbbbbbbbbbbbbbbbb40')]

<_MainThread(MainThread, started 18744)>

[(50, '50boobook50', '50aaaaaaaaaaaa50', '50bbbbbbbbbbbbbbbbb50'), (50, '50boobook50', '50aaaaaaaaaaaa50', '50bbbbbbbbbbbbbbbbb50'), (50, '50boobook50', '50aaaaaaaaaaaa50', '50bbbbbbbbbbbbbbbbb50')]

<_MainThread(MainThread, started 18744)>

[(60, '60boobook60', '60aaaaaaaaaaaa60', '60bbbbbbbbbbbbbbbbb60'), (60, '60boobook60', '60aaaaaaaaaaaa60', '60bbbbbbbbbbbbbbbbb60'), (60, '60boobook60', '60aaaaaaaaaaaa60', '60bbbbbbbbbbbbbbbbb60')]

<_MainThread(MainThread, started 18744)>

<_MainThread(MainThread, started 18744)>

[(80, '80boobook80', '80aaaaaaaaaaaa80', '80bbbbbbbbbbbbbbbbb80'), (80, '80boobook80', '80aaaaaaaaaaaa80', '80bbbbbbbbbbbbbbbbb80'), (80, '80boobook80', '80aaaaaaaaaaaa80', '80bbbbbbbbbbbbbbbbb80')]

<_MainThread(MainThread, started 18744)>

[(90, '90boobook90', '90aaaaaaaaaaaa90', '90bbbbbbbbbbbbbbbbb90'), (90, '90boobook90', '90aaaaaaaaaaaa90', '90bbbbbbbbbbbbbbbbb90'), (90, '90boobook90', '90aaaaaaaaaaaa90', '90bbbbbbbbbbbbbbbbb90')]

<_MainThread(MainThread, started 18744)>

[(100, '100boobook100', '100aaaaaaaaaaaa100', '100bbbbbbbbbbbbbbbbb100'), (100, '100boobook100', '100aaaaaaaaaaaa100', '100bbbbbbbbbbbbbbbbb100'), (100, '100boobook100', '100aaaaaaaaaaaa100', '100bbbbbbbbbbbbbbbbb100')]

all over

Wed Mar 28 11:08:34 2018

Process finished with exit code 0

这个时候是串行 花费了15秒

多线程跑:

#coding=utf-8

import threading

import time

import cx_Oracle

from pprint import pprint

import csv

print time.asctime()

table_name = "dbtest.csv"

f = open(table_name + ".csv", "w")

def query01():

tname = threading.current_thread()

aa=10

conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')

cursor = conn.cursor()

# cursor.execute("SELECT * FROM TEST100 WHERE ID = ％s", [aa])

cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")

cursor.execute(None,{'id':aa})

row = cursor.fetchall()

print row

return row

cursor.close()

conn.close()

def query02():

tname = threading.current_thread()

aa=20

conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')

cursor = conn.cursor()

# cursor.execute("SELECT * FROM TEST100 WHERE ID = ％s", [aa])

cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")

cursor.execute(None,{'id':aa})

row = cursor.fetchall()

print row

return row

cursor.close()

conn.close()

def query03():

tname = threading.current_thread()

aa=30

conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')

cursor = conn.cursor()

# cursor.execute("SELECT * FROM TEST100 WHERE ID = ％s", [aa])

cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")

cursor.execute(None,{'id':aa})

row = cursor.fetchall()

print row

return row

cursor.close()

conn.close()

def query04():

conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')

cursor = conn.cursor()

tname = threading.current_thread()

aa=40

# cursor.execute("SELECT * FROM TEST100 WHERE ID = ％s", [aa])

cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")

cursor.execute(None,{'id':aa})

row = cursor.fetchall()

print row

return row

cursor.close()

conn.close()

def query05():

conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')

cursor = conn.cursor()

tname = threading.current_thread()

aa=50

# cursor.execute("SELECT * FROM TEST100 WHERE ID = ％s", [aa])

cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")

cursor.execute(None,{'id':aa})

row = cursor.fetchall()

print row

return row

cursor.close()

conn.close()

def query06():

conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')

cursor = conn.cursor()

tname = threading.current_thread()

aa=60

# cursor.execute("SELECT * FROM TEST100 WHERE ID = ％s", [aa])

cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")

cursor.execute(None,{'id':aa})

row = cursor.fetchall()

print row

return row

cursor.close()

conn.close()

def query07():

conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')

cursor = conn.cursor()

tname = threading.current_thread()

aa=70

# cursor.execute("SELECT * FROM TEST100 WHERE ID = ％s", [aa])

cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")

cursor.execute(None,{'id':aa})

row = cursor.fetchall()

return row

cursor.close()

conn.close()

def query08():

conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')

cursor = conn.cursor()

tname = threading.current_thread()

aa=80

# cursor.execute("SELECT * FROM TEST100 WHERE ID = ％s", [aa])

cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")

cursor.execute(None,{'id':aa})

row = cursor.fetchall()

print row

return row

cursor.close()

conn.close()

def query09():

conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')

cursor = conn.cursor()

tname = threading.current_thread()

aa=90

# cursor.execute("SELECT * FROM TEST100 WHERE ID = ％s", [aa])

cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")

cursor.execute(None,{'id':aa})

row = cursor.fetchall()

print row

return row

cursor.close()

conn.close()

def query10():

conn = cx_Oracle.connect('tlcbuser/tlcbuser@20.5.101.31/tlyy')

cursor = conn.cursor()

tname = threading.current_thread()

aa=100

# cursor.execute("SELECT * FROM TEST100 WHERE ID = ％s", [aa])

cursor.prepare("""SELECT * FROM TEST100 WHERE ID=:id""")

cursor.execute(None,{'id':aa})

row = cursor.fetchall()

print row

return row

cursor.close()

conn.close()

threads = []

t1 = threading.Thread(target=query01, name='query01')

threads.append(t1)

t2 = threading.Thread(target=query02, name='query02')

threads.append(t2)

t2 = threading.Thread(target=query03, name='query03')

threads.append(t2)

t2 = threading.Thread(target=query04, name='query04')

threads.append(t2)

t2 = threading.Thread(target=query05, name='query05')

threads.append(t2)

t2 = threading.Thread(target=query06, name='query06')

threads.append(t2)

t2 = threading.Thread(target=query07, name='query07')

threads.append(t2)

t2 = threading.Thread(target=query08, name='query08')

threads.append(t2)

t2 = threading.Thread(target=query09, name='query09')

threads.append(t2)

t2 = threading.Thread(target=query10, name='query10')

threads.append(t2)

if __name__ == '__main__':

for t in threads:

# t.setDaemon(True)

t.start()

# t.run()

# t.start()

# print '3333333'

print threading.current_thread()

# print t.is_alive()

# print '3333333'

t.join()

print "all over "

print time.asctime()

C:\Python27\python.exe C:/Users/tlcb/PycharmProjects/untitled/a2.py

Wed Mar 28 11:12:47 2018

<_MainThread(MainThread, started 22500)>

<_MainThread(MainThread, started 22500)>

<_MainThread(MainThread, started 22500)>

<_MainThread(MainThread, started 22500)>

<_MainThread(MainThread, started 22500)>

<_MainThread(MainThread, started 22500)>

<_MainThread(MainThread, started 22500)>

<_MainThread(MainThread, started 22500)>

<_MainThread(MainThread, started 22500)>

<_MainThread(MainThread, started 22500)>

[(40, '40boobook40', '40aaaaaaaaaaaa40', '40bbbbbbbbbbbbbbbbb40'), (40, '40boobook40', '40aaaaaaaaaaaa40', '40bbbbbbbbbbbbbbbbb40'), (40, '40boobook40', '40aaaaaaaaaaaa40', '40bbbbbbbbbbbbbbbbb40')]

[(60, '60boobook60', '60aaaaaaaaaaaa60', '60bbbbbbbbbbbbbbbbb60'), (60, '60boobook60', '60aaaaaaaaaaaa60', '60bbbbbbbbbbbbbbbbb60'), (60, '60boobook60', '60aaaaaaaaaaaa60', '60bbbbbbbbbbbbbbbbb60')]

[(80, '80boobook80', '80aaaaaaaaaaaa80', '80bbbbbbbbbbbbbbbbb80'), (80, '80boobook80', '80aaaaaaaaaaaa80', '80bbbbbbbbbbbbbbbbb80'), (80, '80boobook80', '80aaaaaaaaaaaa80', '80bbbbbbbbbbbbbbbbb80')]

[(50, '50boobook50', '50aaaaaaaaaaaa50', '50bbbbbbbbbbbbbbbbb50'), (50, '50boobook50', '50aaaaaaaaaaaa50', '50bbbbbbbbbbbbbbbbb50'), (50, '50boobook50', '50aaaaaaaaaaaa50', '50bbbbbbbbbbbbbbbbb50')]

[(10, '10boobook10', '10aaaaaaaaaaaa10', '10bbbbbbbbbbbbbbbbb10'), (10, '10sssssssss10', 'tlcb', 'tlcb'), (10, '10boobook10', '10aaaaaaaaaaaa10', '10bbbbbbbbbbbbbbbbb10')]

[(20, '20boobook20', '20aaaaaaaaaaaa20', '20bbbbbbbbbbbbbbbbb20'), (20, '20boobook20', '20aaaaaaaaaaaa20', '20bbbbbbbbbbbbbbbbb20'), (20, '20boobook20', '20aaaaaaaaaaaa20', '20bbbbbbbbbbbbbbbbb20')]

[(30, '30boobook30', '30aaaaaaaaaaaa30', '30bbbbbbbbbbbbbbbbb30'), (30, '30boobook30', '30aaaaaaaaaaaa30', '30bbbbbbbbbbbbbbbbb30'), (30, '30boobook30', '30aaaaaaaaaaaa30', '30bbbbbbbbbbbbbbbbb30')]

[(100, '100boobook100', '100aaaaaaaaaaaa100', '100bbbbbbbbbbbbbbbbb100'), (100, '100boobook100', '100aaaaaaaaaaaa100', '100bbbbbbbbbbbbbbbbb100'), (100, '100boobook100', '100aaaaaaaaaaaa100', '100bbbbbbbbbbbbbbbbb100')]

[(90, '90boobook90', '90aaaaaaaaaaaa90', '90bbbbbbbbbbbbbbbbb90'), (90, '90boobook90', '90aaaaaaaaaaaa90', '90bbbbbbbbbbbbbbbbb90'), (90, '90boobook90', '90aaaaaaaaaaaa90', '90bbbbbbbbbbbbbbbbb90')]

all over

Wed Mar 28 11:12:55 2018

Process finished with exit code 0

此时花了 8秒

补充知识：python logging定制logstash的json日志格式

最近一直在折腾日志的收集，现在算是收尾了。 写一篇算python优化logstash的方案。

其实大家都知道logstash调用grok来解析日志的话，是要消耗cpu的成本的，毕竟是需要正则的匹配的。

根据logstash调优的方案，咱们可以预先生成json的格式。 我这边基本是python的程序，怎么搞尼 ？

有两种方法，第一种方法是生成json后，直接打入logstash的端口。 还有一种是生成json写入文件，让logstash做tail操作的时候，把一行的日志数据直接载入json就可以了。

python下的日志调试用得时logging，改成json也是很好改得。 另外不少老外已经考虑到这样的需求，已经做了python logstash的模块。


import logging
import logstash
import sys

host = 'localhost'

test_logger = logging.getLogger('python-logstash-logger')
test_logger.setLevel(logging.INFO)
test_logger.addHandler(logstash.LogstashHandler(host, 5959, version=1))
# test_logger.addHandler(logstash.TCPLogstashHandler(host, 5959, version=1))

test_logger.error('python-logstash: test logstash error message.')
test_logger.info('python-logstash: test logstash info message.')
test_logger.warning('python-logstash: test logstash warning message.')

# add extra field to logstash message
extra = {
 'test_string': 'python version: ' + repr(sys.version_info),
 'test_boolean': True,
 'test_dict': {'a': 1, 'b': 'c'},
 'test_float': 1.23,
 'test_integer': 123,
 'test_list': [1, 2, '3'],
}
test_logger.info('python-logstash: test extra fields', extra=extra)

python-logstash自带了amqp的方案


import logging
import logstash

# AMQP parameters
host = 'localhost'
username = 'guest'
password= 'guest'
exchange = 'logstash.py'

# get a logger and set logging level
test_logger = logging.getLogger('python-logstash-logger')
test_logger.setLevel(logging.INFO)

# add the handler
test_logger.addHandler(logstash.AMQPLogstashHandler(version=1,
                         host=host,
                         durable=True,
                         username=username,
                         password=password,
                         exchange=exchange))

# log
test_logger.error('python-logstash: test logstash error message.')
test_logger.info('python-logstash: test logstash info message.')
test_logger.warning('python-logstash: test logstash warning message.')

try:
 1/0
except:
 test_logger.exception('python-logstash: test logstash exception with stack trace')

不管怎么说，最后生成的格式是这样就可以了。


{
 "@source"=>"unknown",
 "@type"=>"nginx",
 "@tags"=>[],
 "@fields"=>{
   "remote_addr"=>"192.168.0.1",
   "remote_user"=>"-",
   "body_bytes_sent"=>"13988",
   "request_time"=>"0.122",
   "status"=>"200",
   "request"=>"GET /some/url HTTP/1.1",
   "request_method"=>"GET",
   "http_referrer"=>"http://www.example.org/some/url",
   "http_user_agent"=>"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/21.0.1180.79 Safari/537.1"
 },
 "@timestamp"=>"2012-08-23T10:49:14+02:00"
}

我这里简单提一下，这个模块用的不是很满意，我在python下把日志打成了json字符串，我原本以为会像grok那样，在Es里面，我的这条日志是个字段的结构，而不是这个日志都在message里面…. 我想大家应该明白了我的意思，这样很是不容易在kibana的搜索…

在kibana搜索，我经常上 source:xxx AND level:INFO 结果正像上面描述的那样，整条日志，都在@message里面。

来源：https://blog.csdn.net/zhaoyangjian724/article/details/79724956

标签：python,串行,执行,并行
0
投稿

猜你喜欢

  • Tensorflow与RNN、双向LSTM等的踩坑记录及解决

    2021-04-29 21:25:55

  • 解决MySql8.0 查看事务隔离级别报错的问题

    2024-01-17 23:14:01

  • DjangoRestFramework 使用 simpleJWT 登陆认证完整记录

    2021-03-29 18:34:12

  • Go语言基础go接口用法示例详解

    2024-04-30 10:06:53

  • python中模块的__all__属性详解

    2022-10-16 08:59:18

  • 解决Django cors跨域问题

    2021-07-02 08:42:36

  • MyBatis批量插入/修改/删除MySql数据

    2024-01-18 05:55:11

  • Python判断三段线能否构成三角形的代码

    2023-01-21 19:44:25

  • asp如何对文件进行操作？

    2009-11-20 18:31:00

  • Python字符串、整数、和浮点型数相互转换实例

    2023-09-20 18:05:52

  • python 爬虫 实现增量去重和定时爬取实例

    2022-01-26 01:01:37

  • Python数据分析之获取双色球历史信息的方法示例

    2022-10-05 21:16:34

  • 阿里云 CentOS7.4 安装 Python3.6的方法讲解

    2022-06-12 23:56:13

  • MySQL之主键索引排序失效问题

    2024-01-19 10:38:53

  • Django返回HTML文件的实现方法

    2022-12-25 16:26:16

  • python中pyqtgraph知识点总结

    2022-02-23 10:24:30

  • mysql跨数据库复制表(在同一IP地址中)示例

    2024-01-20 00:51:11

  • Python+OpenCV实战之利用 K-Means 聚类进行色彩量化

    2021-01-02 09:20:27

  • Django uwsgi Nginx 的生产环境部署详解

    2023-10-21 14:55:30

  • Jmail发信的实例,模块化随时调用

    2007-09-27 13:35:00
    • asp之家 网络编程 m.aspxhome.com