Merge pull request #192 from dalf/connection-pool

[enh] improve response time. close #100
This commit is contained in:
Adam Tauber 2015-01-21 19:44:20 +01:00
commit 10891bdeab
5 changed files with 65 additions and 5 deletions

View File

@ -17,13 +17,13 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
from lxml import etree from lxml import etree
from requests import get
from json import loads from json import loads
from urllib import urlencode from urllib import urlencode
from searx.languages import language_codes from searx.languages import language_codes
from searx.engines import ( from searx.engines import (
categories, engines, engine_shortcuts categories, engines, engine_shortcuts
) )
from searx.poolrequests import get
def searx_bang(full_query): def searx_bang(full_query):

View File

@ -1,6 +1,6 @@
import json import json
from requests import get
from urllib import urlencode from urllib import urlencode
from searx.poolrequests import get
from searx.utils import format_date_by_locale from searx.utils import format_date_by_locale
result_count = 1 result_count = 1

61
searx/poolrequests.py Normal file
View File

@ -0,0 +1,61 @@
import requests
the_http_adapter = requests.adapters.HTTPAdapter(pool_connections=100)
the_https_adapter = requests.adapters.HTTPAdapter(pool_connections=100)
class SessionSinglePool(requests.Session):
def __init__(self):
global the_https_adapter, the_http_adapter
super(SessionSinglePool, self).__init__()
# reuse the same adapters
self.adapters.clear()
self.mount('https://', the_https_adapter)
self.mount('http://', the_http_adapter)
def close(self):
"""Call super, but clear adapters since there are managed globaly"""
self.adapters.clear()
super(SessionSinglePool, self).close()
def request(method, url, **kwargs):
"""same as requests/requests/api.py request(...) except it use SessionSinglePool"""
session = SessionSinglePool()
response = session.request(method=method, url=url, **kwargs)
session.close()
return response
def get(url, **kwargs):
kwargs.setdefault('allow_redirects', True)
return request('get', url, **kwargs)
def options(url, **kwargs):
kwargs.setdefault('allow_redirects', True)
return request('options', url, **kwargs)
def head(url, **kwargs):
kwargs.setdefault('allow_redirects', False)
return request('head', url, **kwargs)
def post(url, data=None, json=None, **kwargs):
return request('post', url, data=data, json=json, **kwargs)
def put(url, data=None, **kwargs):
return request('put', url, data=data, **kwargs)
def patch(url, data=None, **kwargs):
return request('patch', url, data=data, **kwargs)
def delete(url, **kwargs):
return request('delete', url, **kwargs)

View File

@ -15,9 +15,9 @@ along with searx. If not, see < http://www.gnu.org/licenses/ >.
(C) 2013- by Adam Tauber, <asciimoo@gmail.com> (C) 2013- by Adam Tauber, <asciimoo@gmail.com>
''' '''
import requests as requests_lib
import threading import threading
import re import re
import searx.poolrequests as requests_lib
from itertools import izip_longest, chain from itertools import izip_longest, chain
from operator import itemgetter from operator import itemgetter
from Queue import Queue from Queue import Queue
@ -31,7 +31,6 @@ from searx.utils import gen_useragent
from searx.query import Query from searx.query import Query
from searx import logger from searx import logger
logger = logger.getChild('search') logger = logger.getChild('search')
number_of_searches = 0 number_of_searches = 0

View File

@ -28,7 +28,6 @@ import os
import hashlib import hashlib
from datetime import datetime, timedelta from datetime import datetime, timedelta
from requests import get as http_get
from itertools import chain from itertools import chain
from urllib import urlencode from urllib import urlencode
from flask import ( from flask import (
@ -37,6 +36,7 @@ from flask import (
) )
from flask.ext.babel import Babel, gettext, format_date from flask.ext.babel import Babel, gettext, format_date
from searx import settings, searx_dir from searx import settings, searx_dir
from searx.poolrequests import get as http_get
from searx.engines import ( from searx.engines import (
categories, engines, get_engines_stats, engine_shortcuts categories, engines, get_engines_stats, engine_shortcuts
) )