http协议的性能测试方案，真正的并发请求

方案1. 普通python

import os
import threading
import binascii
import urllib2

import gflags
FLAGS = gflags.FLAGS
gflags.DEFINE_string('url',"http://192.168.1.229:8080/vt","api")
gflags.DEFINE_integer('num_processes', 0, "num of processes")
gflags.DEFINE_integer('num_threads', 4, "num of threads per process")
gflags.DEFINE_integer('count', 5000, "num of access per thread")

try:
    import multiprocessing # Python 2.6+
except ImportError:
    multiprocessing = None

def _cpu_count():
    if multiprocessing is not None:
        try:
            return multiprocessing.cpu_count()
        except NotImplementedError:
            pass
    try:
        return os.sysconf("SC_NPROCESSORS_CONF")
    except ValueError:
        pass
    print "Could not detect number of processors; running with one process"
    return 1


def task_worker(url, count):
    arg = "nt=3&eip=210.252.123.141&eport=27017&evt=0"
    while count:
        tid = binascii.b2a_hex(os.urandom(20))
        pid = binascii.b2a_hex(os.urandom(20))
        param = arg + "&pid=" + pid
        api = url + "/" + tid + "?" + param
        try:
            f = urllib2.urlopen(api)
        except urllib2.HTTPError, e:
            print "access tracker error"
        count -= 1

if __name__ == "__main__":
    import sys
    try:
        argv = FLAGS(sys.argv)
    except gflags.FlagsError, e:
        print 'parse arg error:', e
        sys.exit(1)

    url = FLAGS.url
    num_processes = FLAGS.num_processes
    num_threads = FLAGS.num_threads
    count = FLAGS.count
    if num_processes <= 0:
        num_processes = _cpu_count()
    if num_processes > 1:
            print "Pre-forking %d server processes" % (num_processes)
            for i in range(num_processes):
                if os.fork() == 0:
                    threads = []
                    for i in range(num_threads):
                        threads.append(threading.Thread(target=task_worker, args=(url,count )))
                    for t in threads:
                        t.start()
                    for t in threads:
                        t.join()
                else:
                    os.waitpid(-1, 0)
    else:
        threads = []
        for i in range(num_threads):
            threads.append(threading.Thread(target=task_worker, args=(url,count )))
        for t in threads:
            t.start()
        for t in threads:
            t.join()

方案2.stackless

    import stackless
import random
import sys

class hackysacker:
    counter = 0
    def __init__(self,name,circle):
        self.name = name
        self.circle = circle
        circle.append(self)
        self.channel = stackless.channel()
        stackless.tasklet(self.messageLoop)()

    def incrementCounter(self):
        hackysacker.counter += 1
        if hackysacker.counter >= turns:
            while self.circle:
                self.circle.pop().channel.send('exit')

    def messageLoop(self):
        while 1:
            message = self.channel.receive()
            if message == 'exit':
                return
            debugPrint("%s got hackeysack from %s" % (self.name, message.name))
            kickTo = self.circle[random.randint(0,len(self.circle)-1)]
            while kickTo is self:
                kickTo = self.circle[random.randint(0,len(self.circle)-1)]
            debugPrint("%s kicking hackeysack to %s" % (self.name, kickTo.name))
            self.incrementCounter()
            kickTo.channel.send(self)
               


def debugPrint(x):
    if debug:print x

debug = 1
hackysackers = 10
turns = 10

def runit(hs=5,ts=5,dbg=1):
    global hackysackers,turns,debug
    hackysackers = hs
    turns = ts
    debug = dbg
   
    hackysacker.counter = 0
    circle = []
    ne = hackysacker('1',circle)

    for i in range(hackysackers):
        hackysacker(`i`,circle)

    one.channel.send(one)

    try:
        stackless.run()
    except TaskletExit:
        pass
   

if __name__ == "__main__":
    runit()

#######################

import stackless
import urllib2
import urllib
import time
import random
import binascii
import os
import httplib
import gflags

FLAGS = gflags.FLAGS
gflags.DEFINE_integer('count', 20000, "num of access per thread")

# return num of processes if  more than one cpu avalible
def _cpu_count():
    try:
        import multiprocessing # Python 2.6+
    except ImportError:
        multiprocessing = None
       
    if multiprocessing is not None:
        try:
            return multiprocessing.cpu_count()
        except NotImplementedError:
            pass
           
    try:
        return os.sysconf("SC_NPROCESSORS_CONF")
    except ValueError:
        pass
    print "Could not detect number of processors; running with one process"
    return 1   


# using stackless schedule: 1. all user connection to server, 2. then request at one time
def openURL(x):
    url = "192.168.1.229"
    arg = "nt=3&eip=210.252.123.141&eport=27017&evt=0"
    tid = binascii.b2a_hex(os.urandom(20))
    pid = binascii.b2a_hex(os.urandom(20))
    param = arg + "&pid=" + pid
    api =  "/" + tid + "?" + param
   
   
    conn = httplib.HTTPConnection(url,80)
    print "numbers connection per", x
   
    stackless.schedule()
   
    start_time = time.time()
    try:
        conn.request("GET", "/vt"+ api)
        print "numbers request per", x

    except URLError, e:
        print e.reason
        print e.code
        print e.read()
   
    finish_time = time.time()
    api_reaction_time_ = (finish_time - start_time)   
    print 'numbers request need time is', api_reaction_time_
   
    stackless.schedule()
   
    res = conn.getresponse()
   
    conn.close()
   
   
   
def task_worker():
    count = FLAGS.count

    for i in xrange(count):
        stackless.tasklet(openURL)(i)
       
    try:
        stackless.run()
    except TaskletExit:
        pass
   
if __name__ == '__main__':
    task_worker()

#########################

import stackless
import urllib2
import urllib
import time
import random
import binascii
import os
import httplib
import gflags

FLAGS = gflags.FLAGS
gflags.DEFINE_integer('count', 20, "num of access per thread")

# return num of processes if  more than one cpu avalible
def _cpu_count():
    try:
        import multiprocessing # Python 2.6+
    except ImportError:
        multiprocessing = None
       
    if multiprocessing is not None:
        try:
            return multiprocessing.cpu_count()
        except NotImplementedError:
            pass
           
    try:
        return os.sysconf("SC_NPROCESSORS_CONF")
    except ValueError:
        pass
    print "Could not detect number of processors; running with one process"
    return 1   


# using stackless schedule: 1. all user connection to server, 2. then request at one time
def openURL(x):
    url = "http://192.168.1.229:8080/vt"
    arg = "nt=3&eip=210.252.123.141&eport=27017&evt=0"
    tid = binascii.b2a_hex(os.urandom(20))
    pid = binascii.b2a_hex(os.urandom(20))
    param = arg + "&pid=" + pid
    api =  "/" + tid + "?" + param
   
    try:
        start_time = time.time()
        req = urllib2.Request(api)
        end_time = time.time()
        print "using time  = ", end_time - start_time
        print "numbers connection per", x

       
    except:
        print "connect errors"
       


       
   
   
def task_worker():
    count = FLAGS.count

    for i in xrange(count):
        stackless.tasklet(openURL)(i)

    try:
        stackless.run()
    except TaskletExit:
        pass
   
if __name__ == '__main__':
    task_worker()

安装步骤：

1.      https://github.com/ml31415/geventhttpclient下载源文件

2.      进入源文件执行python setup.py install,返回错误error: Unable to find vcvarsall.bat

3.      MinGW，VC2010，添加环境变量

a)        C:\MinGW\mingw32\bin

b)        C:\MinGW\bin

c)        C:\Python27;C:\Python27\Scripts;

d)        C:\Program Files\Microsoft SDKs\Windows\v7.0A\bin;

4.      啊啊之后在python安装路径的Lib\distutils增加distutils.cfg，

[build]

compiler=mingw32

5.      Scc1.exe: error: unrecognized command line option '-mno-cygwin',

6.      然后对Lib\distutils\cygwinccompiler.py动手，把里面的-mno-cygwin都去掉（可能去掉适当的一两个就是了，不过俺小白，于是都去掉，事后恢复）；

7. setup.py install build --compiler=mingw32

8.      如果不行试试pip或者easy_install

import gevent
import json
from geventhttpclient import HTTPClient
from geventhttpclient.url import URL
import binascii
import os
import time

def openURL(i):

    while True:
        url = "http://192.168.1.229/vt"
        arg = "nt=3&eip=210.252.123.141&eport=27017&evt=0"
        tid = binascii.b2a_hex(os.urandom(20))
        pid = binascii.b2a_hex(os.urandom(20))
        param = arg + "&pid=" + pid
        api =  url + "/" + tid +'?' +param
        url = URL(api)
        print i,'line url as below'
        print url
        http = HTTPClient.from_url(url)
        response = http.get(url.request_uri)
        print i, 'line url response as below',
        print response.status_code

        http.close()
   
def task_worker():

    # allow to run 20 greenlet at a time, this is more than concurrency
    # of the http client but isn't a problem since the client has its own
    # connection pool.
    pool = gevent.pool.Pool(5000)
    for i in xrange(5000):
        pool.spawn(openURL,i)

    pool.join()

   
if __name__ == '__main__':
    task_worker()

方案4. httpperf

http://www.hpl.hp.com/research/linux/httperf/

http://code.google.com/p/httperf/

1.      安装httperf在linux

2.      配置ulimit的参数

（http://blog.163.com/dyc_888@126/blog/static/10044335120111122101855101/）

  vim /etc/security/limits.conf
     #添加如下的行

    * soft nofile 4100
    * hard nofile 4100

3.      Python运动命令

httperf请求命令参数：

httperf --timeout=5 --server='192.168.1.127' --port=80 --uri='/vt/4ffba77180d008304aa1bc9737e1dfbdbb6b245d?pid=4ffba77180d008304aa1bc9737e1dfbdbb6b245d&eip=1.1.1.1&eport=80&nt=3&evt=0' --num-conns=5000 --num-calls=20 --rate=200

httperf运行结果：

httperf --timeout=5 --client=0/1 --server=192.168.1.127 --port=80 --uri=/vt/4ffba77180d008304aa1bc9737e1dfbdbb6b245d?pid=4ffba77180d008304aa1bc9737e1dfbdbb6b245d&eip=1.1.1.1&eport=80&nt=3&evt=0 --rate=200 --send-buffer=4096 --recv-buffer=16384 --num-conns=5000 --num-calls=20

Maximum connect burst length: 8

Total:connections 5000requests 100000 replies 100000 test-duration 25.432 s

解释：connections 5000=--num-coons, requests 100000=--num-conns * --num-call

Connection rate:196.6 conn/s (5.1 ms/conn, <=60 concurrent connections)

Connection time [ms]: min 84.5 avg 169.2 max 3214.3 median 164.5 stddev 71.1

Connection time [ms]: connect 10.2

Connection length [replies/conn]: 20.000