modified fetch function with semaphore

import random
import asyncio
from aiohttp import ClientSession

async def fetch(url, session):
    async with session.get(url) as response:
        delay = response.headers.get("DELAY")
        date = response.headers.get("DATE")
        print("{}:{} with delay {}".format(date, response.url, delay))
        return await response.read()


async def bound_fetch(sem, url, session):
    # Getter function with semaphore.
    async with sem:
        await fetch(url, session)

我在Windows上跑，在资源管理器看到创建了25个线程 async def run(r): url = "http://localhost:8080/{}" tasks = [] # create instance of Semaphore sem = asyncio.Semaphore(1000)

    # Create client session that will ensure we dont open new connection
    # per each request.
    async with ClientSession() as session:
        for i in range(r):
            # pass Semaphore and session to every GET request
            task = asyncio.ensure_future(bound_fetch(sem, url.format(i), session))
            tasks.append(task)

        responses = asyncio.gather(*tasks)
        await responses

number = 10000
loop = asyncio.get_event_loop()

future = asyncio.ensure_future(run(number))
loop.run_until_complete(future)

modified fetch function with semaphore

import random
import asyncio
from aiohttp import ClientSession

async def fetch(url, session):
    async with session.get(url) as response:
        delay = response.headers.get("DELAY")
        date = response.headers.get("DATE")
        print("{}:{} with delay {}".format(date, response.url, delay))
        return await response.read()


async def bound_fetch(sem, url, session):
    # Getter function with semaphore.
    async with sem:
        await fetch(url, session)


async def run(r):
    url = "http://localhost:8080/{}"
    tasks = []
    # create instance of Semaphore
    sem = asyncio.Semaphore(1000)

    # Create client session that will ensure we dont open new connection
    # per each request.
    async with ClientSession() as session:
        for i in range(r):
            # pass Semaphore and session to every GET request
            task = asyncio.ensure_future(bound_fetch(sem, url.format(i), session))
            tasks.append(task)

        responses = asyncio.gather(*tasks)
        await responses

number = 10000
loop = asyncio.get_event_loop()

future = asyncio.ensure_future(run(number))
loop.run_until_complete(future)

我在Windows的资源管理器看的线程数，刚才中间怎么打断了。。。

gevent:

import gevent.monkey
gevent.monkey.patch_socket()

import gevent
import urllib.request as req
import json

def fetch(pid):
    response = req.urlopen('http://localhost:8080/')
    result = response.read()
    #json_result = json.loads(result)
    #datetime = json_result['datetime']

    print('Process %s' % (pid))
    return result

def asynchronous():
    threads = []
    for i in range(1,800):
        threads.append(gevent.spawn(fetch, i))
    gevent.joinall(threads)

print('Asynchronous:')
asynchronous()

gevent打了monkey patch也是创建15线程呀。。

自己结帖，测试数据为：Windows下用asyncio跑一个run_forever()要开5个线程，用aiohttp发起请求最大创建25个线程，使用基于线程池的DNS resolver的gevent发起请求创建15个线程。而Linux（Ubuntu）下用asyncio跑空的事件循环只用1个线程，用aiohttp发起请求也只用1个线程，使用基于线程池DNS resolver的gevent创建10个线程，而在设置export GEVENT_RESOLVER=ares后也是只开一个线程，在Linux下的行为符合我对基于协程的异步IO的理解，而Windows下ayncio库为什么要创建这么多线程还是不理解，google也搜不到什么信息，望有大神告知。

不对，刚再测了一下aiohttp发起请求还是要创建5个线程的，但也比Windows下创建少的多。。