http协议的性能测试方案,真正的并发请求
上一篇 / 下一篇 2014-01-15 11:49:13 / 个人分类:python
方案1. 普通python
import os
import threading
import binascii
import urllib2
import gflags
FLAGS = gflags.FLAGS
gflags.DEFINE_string('url',"http://192.168.1.229:8080/vt","api")
gflags.DEFINE_integer('num_processes', 0, "num of processes")
gflags.DEFINE_integer('num_threads', 4, "num of threads per process")
gflags.DEFINE_integer('count', 5000, "num of access per thread")
try:
import multiprocessing # Python 2.6+
except ImportError:
multiprocessing = None
def _cpu_count():
if multiprocessing is not None:
try:
return multiprocessing.cpu_count()
except NotImplementedError:
pass
try:
return os.sysconf("SC_NPROCESSORS_CONF")
except ValueError:
pass
print "Could not detect number of processors; running with one process"
return 1
def task_worker(url, count):
arg = "nt=3&eip=210.252.123.141&eport=27017&evt=0"
while count:
tid = binascii.b2a_hex(os.urandom(20))
pid = binascii.b2a_hex(os.urandom(20))
param = arg + "&pid=" + pid
api = url + "/" + tid + "?" + param
try:
f = urllib2.urlopen(api)
except urllib2.HTTPError, e:
print "access tracker error"
count -= 1
if __name__ == "__main__":
import sys
try:
argv = FLAGS(sys.argv)
except gflags.FlagsError, e:
print 'parse arg error:', e
sys.exit(1)
url = FLAGS.url
num_processes = FLAGS.num_processes
num_threads = FLAGS.num_threads
count = FLAGS.count
if num_processes <= 0:
num_processes = _cpu_count()
if num_processes > 1:
print "Pre-forking %d server processes" % (num_processes)
for i in range(num_processes):
if os.fork() == 0:
threads = []
for i in range(num_threads):
threads.append(threading.Thread(target=task_worker, args=(url,count )))
for t in threads:
t.start()
for t in threads:
t.join()
else:
os.waitpid(-1, 0)
else:
threads = []
for i in range(num_threads):
threads.append(threading.Thread(target=task_worker, args=(url,count )))
for t in threads:
t.start()
for t in threads:
t.join()
方案2.stackless
import stackless
import random
import sys
class hackysacker:
counter = 0
def __init__(self,name,circle):
self.name = name
self.circle = circle
circle.append(self)
self.channel = stackless.channel()
stackless.tasklet(self.messageLoop)()
def incrementCounter(self):
hackysacker.counter += 1
if hackysacker.counter >= turns:
while self.circle:
self.circle.pop().channel.send('exit')
def messageLoop(self):
while 1:
message = self.channel.receive()
if message == 'exit':
return
debugPrint("%s got hackeysack from %s" % (self.name, message.name))
kickTo = self.circle[random.randint(0,len(self.circle)-1)]
while kickTo is self:
kickTo = self.circle[random.randint(0,len(self.circle)-1)]
debugPrint("%s kicking hackeysack to %s" % (self.name, kickTo.name))
self.incrementCounter()
kickTo.channel.send(self)
def debugPrint(x):
if debug:print x
debug = 1
hackysackers = 10
turns = 10
def runit(hs=5,ts=5,dbg=1):
global hackysackers,turns,debug
hackysackers = hs
turns = ts
debug = dbg
hackysacker.counter = 0
circle = []
ne = hackysacker('1',circle)
for i in range(hackysackers):
hackysacker(`i`,circle)
one.channel.send(one)
try:
stackless.run()
except TaskletExit:
pass
if __name__ == "__main__":
runit()
#######################
import stackless
import urllib2
import urllib
import time
import random
import binascii
import os
import httplib
import gflags
FLAGS = gflags.FLAGS
gflags.DEFINE_integer('count', 20000, "num of access per thread")
# return num of processes if more than one cpu avalible
def _cpu_count():
try:
import multiprocessing # Python 2.6+
except ImportError:
multiprocessing = None
if multiprocessing is not None:
try:
return multiprocessing.cpu_count()
except NotImplementedError:
pass
try:
return os.sysconf("SC_NPROCESSORS_CONF")
except ValueError:
pass
print "Could not detect number of processors; running with one process"
return 1
# using stackless schedule: 1. all user connection to server, 2. then request at one time
def openURL(x):
url = "192.168.1.229"
arg = "nt=3&eip=210.252.123.141&eport=27017&evt=0"
tid = binascii.b2a_hex(os.urandom(20))
pid = binascii.b2a_hex(os.urandom(20))
param = arg + "&pid=" + pid
api = "/" + tid + "?" + param
conn = httplib.HTTPConnection(url,80)
print "numbers connection per", x
stackless.schedule()
start_time = time.time()
try:
conn.request("GET", "/vt"+ api)
print "numbers request per", x
except URLError, e:
print e.reason
print e.code
print e.read()
finish_time = time.time()
api_reaction_time_ = (finish_time - start_time)
print 'numbers request need time is', api_reaction_time_
stackless.schedule()
res = conn.getresponse()
conn.close()
def task_worker():
count = FLAGS.count
for i in xrange(count):
stackless.tasklet(openURL)(i)
try:
stackless.run()
except TaskletExit:
pass
if __name__ == '__main__':
task_worker()
#########################
import stackless
import urllib2
import urllib
import time
import random
import binascii
import os
import httplib
import gflags
FLAGS = gflags.FLAGS
gflags.DEFINE_integer('count', 20, "num of access per thread")
# return num of processes if more than one cpu avalible
def _cpu_count():
try:
import multiprocessing # Python 2.6+
except ImportError:
multiprocessing = None
if multiprocessing is not None:
try:
return multiprocessing.cpu_count()
except NotImplementedError:
pass
try:
return os.sysconf("SC_NPROCESSORS_CONF")
except ValueError:
pass
print "Could not detect number of processors; running with one process"
return 1
# using stackless schedule: 1. all user connection to server, 2. then request at one time
def openURL(x):
url = "http://192.168.1.229:8080/vt"
arg = "nt=3&eip=210.252.123.141&eport=27017&evt=0"
tid = binascii.b2a_hex(os.urandom(20))
pid = binascii.b2a_hex(os.urandom(20))
param = arg + "&pid=" + pid
api = "/" + tid + "?" + param
try:
start_time = time.time()
req = urllib2.Request(api)
end_time = time.time()
print "using time = ", end_time - start_time
print "numbers connection per", x
except:
print "connect errors"
def task_worker():
count = FLAGS.count
for i in xrange(count):
stackless.tasklet(openURL)(i)
try:
stackless.run()
except TaskletExit:
pass
if __name__ == '__main__':
task_worker()
安装步骤:
1. https://github.com/ml31415/geventhttpclient下载源文件
2. 进入源文件执行python setup.py install,返回错误error: Unable to find vcvarsall.bat
3. MinGW,VC2010,添加环境变量
a) C:\MinGW\mingw32\bin
b) C:\MinGW\bin
c) C:\Python27;C:\Python27\Scripts;
d) C:\Program Files\Microsoft SDKs\Windows\v7.0A\bin;
4. 啊啊之后在python安装路径的Lib\distutils增加distutils.cfg,
[build]
compiler=mingw32
5. Scc1.exe: error: unrecognized command line option '-mno-cygwin',
6. 然后对Lib\distutils\cygwinccompiler.py动手,把里面的-mno-cygwin都去掉(可能去掉适当的一两个就是了,不过俺小白,于是都去掉,事后恢复);
7. setup.py install build --compiler=mingw32
8. 如果不行试试pip或者easy_install
import gevent
import json
from geventhttpclient import HTTPClient
from geventhttpclient.url import URL
import binascii
import os
import time
def openURL(i):
while True:
url = "http://192.168.1.229/vt"
arg = "nt=3&eip=210.252.123.141&eport=27017&evt=0"
tid = binascii.b2a_hex(os.urandom(20))
pid = binascii.b2a_hex(os.urandom(20))
param = arg + "&pid=" + pid
api = url + "/" + tid +'?' +param
url = URL(api)
print i,'line url as below'
print url
http = HTTPClient.from_url(url)
response = http.get(url.request_uri)
print i, 'line url response as below',
print response.status_code
http.close()
def task_worker():
# allow to run 20 greenlet at a time, this is more than concurrency
# of the http client but isn't a problem since the client has its own
# connection pool.
pool = gevent.pool.Pool(5000)
for i in xrange(5000):
pool.spawn(openURL,i)
pool.join()
if __name__ == '__main__':
task_worker()
方案4. httpperf
http://www.hpl.hp.com/research/linux/httperf/
http://code.google.com/p/httperf/
1. 安装httperf在linux
2. 配置ulimit的参数
(http://blog.163.com/dyc_888@126/blog/static/10044335120111122101855101/)
vim /etc/security/limits.conf
#添加如下的行
* soft nofile 4100
* hard nofile 4100
3. Python运动命令
httperf请求命令参数:
httperf --timeout=5 --server='192.168.1.127' --port=80 --uri='/vt/4ffba77180d008304aa1bc9737e1dfbdbb6b245d?pid=4ffba77180d008304aa1bc9737e1dfbdbb6b245d&eip=1.1.1.1&eport=80&nt=3&evt=0' --num-conns=5000 --num-calls=20 --rate=200
httperf运行结果:
httperf --timeout=5 --client=0/1 --server=192.168.1.127 --port=80 --uri=/vt/4ffba77180d008304aa1bc9737e1dfbdbb6b245d?pid=4ffba77180d008304aa1bc9737e1dfbdbb6b245d&eip=1.1.1.1&eport=80&nt=3&evt=0 --rate=200 --send-buffer=4096 --recv-buffer=16384 --num-conns=5000 --num-calls=20
Maximum connect burst length: 8
Total:connections 5000requests 100000 replies 100000 test-duration 25.432 s
解释:connections 5000=--num-coons, requests 100000=--num-conns * --num-call
Connection rate:196.6 conn/s (5.1 ms/conn, <=60 concurrent connections)
Connection time [ms]: min 84.5 avg 169.2 max 3214.3 median 164.5 stddev 71.1
Connection time [ms]: connect 10.2
Connection length [replies/conn]: 20.000