projects/aports·blob master · refs

553 lines · 19.9 KiB

1Patch-Source: https://github.com/wikier/keepalive/pull/11
2From 64393f6c5bf9c69d946b584fd664dd4df72604e6 Mon Sep 17 00:00:00 2001
3From: Theodore Ni <[email protected]>
4Date: Tue, 6 Jun 2023 21:51:59 -0700
5Subject: [PATCH] Convert to Python 3 and drop support for Python 2.
6
7---
8 README.md | 2 +-
9 keepalive/keepalive.py | 226 +++++++++++++++++------------------------
10 setup.py | 28 ++---
11 3 files changed, 103 insertions(+), 153 deletions(-)
12
13diff --git a/keepalive/keepalive.py b/keepalive/keepalive.py
14index c07ef81..43787c7 100644
15--- a/keepalive/keepalive.py
16+++ b/keepalive/keepalive.py
17@@ -20,15 +20,15 @@
18 # Copyright 2002-2004 Michael D. Stenner, Ryan Tomayko
19 # Copyright 2015 Sergio Fernández
20
21-"""An HTTP handler for urllib2 that supports HTTP 1.1 and keepalive.
22+"""An HTTP handler for urllib that supports HTTP 1.1 and keepalive.
23
24->>> import urllib2
25+>>> import urllib.request
26 >>> from keepalive import HTTPHandler
27 >>> keepalive_handler = HTTPHandler()
28->>> opener = urllib2.build_opener(keepalive_handler)
29->>> urllib2.install_opener(opener)
30+>>> opener = urllib.request.build_opener(keepalive_handler)
31+>>> urllib.request.install_opener(opener)
32 >>>
33->>> fo = urllib2.urlopen('http://www.python.org')
34+>>> fo = urllib.request.urlopen('http://www.python.org')
35
36 If a connection to a given host is requested, and all of the existing
37 connections are still in use, another connection will be opened. If
38@@ -58,7 +58,7 @@
39
40 Upon a status of 200, the object returned has a few additional
41 attributes and methods, which should not be used if you want to
42- remain consistent with the normal urllib2-returned objects:
43+ remain consistent with the normal urllib-returned objects:
44
45 close_connection() - close the connection to the host
46 readlines() - you know, readlines()
47@@ -73,55 +73,30 @@
48
49 Unfortunately, these are ONLY there if status == 200, so it's not
50 easy to distinguish between non-200 responses. The reason is that
51- urllib2 tries to do clever things with error codes 301, 302, 401,
52+ urllib tries to do clever things with error codes 301, 302, 401,
53 and 407, and it wraps the object upon return.
54
55- For python versions earlier than 2.4, you can avoid this fancy error
56- handling by setting the module-level global HANDLE_ERRORS to zero.
57- You see, prior to 2.4, it's the HTTP Handler's job to determine what
58- to handle specially, and what to just pass up. HANDLE_ERRORS == 0
59- means "pass everything up". In python 2.4, however, this job no
60- longer belongs to the HTTP Handler and is now done by a NEW handler,
61- HTTPErrorProcessor. Here's the bottom line:
62-
63- python version < 2.4
64- HANDLE_ERRORS == 1 (default) pass up 200, treat the rest as
65- errors
66- HANDLE_ERRORS == 0 pass everything up, error processing is
67- left to the calling code
68- python version >= 2.4
69- HANDLE_ERRORS == 1 pass up 200, treat the rest as errors
70- HANDLE_ERRORS == 0 (default) pass everything up, let the
71- other handlers (specifically,
72- HTTPErrorProcessor) decide what to do
73-
74- In practice, setting the variable either way makes little difference
75- in python 2.4, so for the most consistent behavior across versions,
76- you probably just want to use the defaults, which will give you
77- exceptions on errors.
78-
79 """
80
81 # $Id: keepalive.py,v 1.17 2006/12/08 00:14:16 mstenner Exp $
82
83-import urllib2
84-import httplib
85+import urllib.error
86+import urllib.request
87+import http.client
88 import socket
89-import thread
90+import _thread
91
92 DEBUG = None
93
94 import sys
95-if sys.version_info < (2, 4): HANDLE_ERRORS = 1
96-else: HANDLE_ERRORS = 0
97-
98+
99 class ConnectionManager:
100 """
101 The connection manager must be able to:
102 * keep track of all existing
103 """
104 def __init__(self):
105- self._lock = thread.allocate_lock()
106+ self._lock = _thread.allocate_lock()
107 self._hostmap = {} # map hosts to a list of connections
108 self._connmap = {} # map connections to host
109 self._readymap = {} # map connection to ready state
110@@ -129,7 +104,7 @@ def __init__(self):
111 def add(self, host, connection, ready):
112 self._lock.acquire()
113 try:
114- if not self._hostmap.has_key(host): self._hostmap[host] = []
115+ if host not in self._hostmap: self._hostmap[host] = []
116 self._hostmap[host].append(connection)
117 self._connmap[connection] = host
118 self._readymap[connection] = ready
119@@ -159,7 +134,7 @@ def get_ready_conn(self, host):
120 conn = None
121 self._lock.acquire()
122 try:
123- if self._hostmap.has_key(host):
124+ if host in self._hostmap:
125 for c in self._hostmap[host]:
126 if self._readymap[c]:
127 self._readymap[c] = 0
128@@ -183,7 +158,7 @@ def __init__(self):
129 def open_connections(self):
130 """return a list of connected hosts and the number of connections
131 to each. [('foo.com:80', 2), ('bar.org', 1)]"""
132- return [(host, len(li)) for (host, li) in self._cm.get_all().items()]
133+ return [(host, len(li)) for (host, li) in list(self._cm.get_all().items())]
134
135 def close_connection(self, host):
136 """close connection(s) to <host>
137@@ -195,7 +170,7 @@ def close_connection(self, host):
138
139 def close_all(self):
140 """close all open connections"""
141- for host, conns in self._cm.get_all().items():
142+ for host, conns in list(self._cm.get_all().items()):
143 for h in conns:
144 self._cm.remove(h)
145 h.close()
146@@ -213,7 +188,7 @@ def _remove_connection(self, host, connection, close=0):
147 def do_open(self, req):
148 host = req.host
149 if not host:
150- raise urllib2.URLError('no host given')
151+ raise urllib.error.URLError('no host given')
152
153 try:
154 h = self._cm.get_ready_conn(host)
155@@ -237,8 +212,8 @@ def do_open(self, req):
156 self._cm.add(host, h, 0)
157 self._start_transaction(h, req)
158 r = h.getresponse()
159- except (socket.error, httplib.HTTPException), err:
160- raise urllib2.URLError(err)
161+ except (socket.error, http.client.HTTPException) as err:
162+ raise urllib.error.URLError(err)
163
164 if DEBUG: DEBUG.info("STATUS: %s, %s", r.status, r.reason)
165
166@@ -255,11 +230,7 @@ def do_open(self, req):
167 r.headers = r.msg
168 r.msg = r.reason
169
170- if r.status == 200 or not HANDLE_ERRORS:
171- return r
172- else:
173- return self.parent.error('http', req, r,
174- r.status, r.msg, r.headers)
175+ return r
176
177 def _reuse_connection(self, h, req, host):
178 """start the transaction with a re-used connection
179@@ -273,7 +244,7 @@ def _reuse_connection(self, h, req, host):
180 r = h.getresponse()
181 # note: just because we got something back doesn't mean it
182 # worked. We'll check the version below, too.
183- except (socket.error, httplib.HTTPException):
184+ except (socket.error, http.client.HTTPException):
185 r = None
186 except:
187 # adding this block just in case we've missed
188@@ -292,7 +263,7 @@ def _reuse_connection(self, h, req, host):
189 raise
190
191 if r is None or r.version == 9:
192- # httplib falls back to assuming HTTP 0.9 if it gets a
193+ # http.client falls back to assuming HTTP 0.9 if it gets a
194 # bad header back. This is most likely to happen if
195 # the socket has been closed by the server since we
196 # last used the connection.
197@@ -312,22 +283,22 @@ def _start_transaction(self, h, req):
198 h.putrequest('POST', req.selector)
199 else:
200 h.putrequest('POST', req.get_selector())
201- if not req.headers.has_key('Content-type'):
202+ if 'Content-type' not in req.headers:
203 h.putheader('Content-type',
204 'application/x-www-form-urlencoded')
205- if not req.headers.has_key('Content-length'):
206+ if 'Content-length' not in req.headers:
207 h.putheader('Content-length', '%d' % len(data))
208 else:
209 if hasattr(req, 'selector'):
210 h.putrequest('GET', req.selector)
211 else:
212 h.putrequest('GET', req.get_selector())
213- except (socket.error, httplib.HTTPException), err:
214- raise urllib2.URLError(err)
215+ except (socket.error, http.client.HTTPException) as err:
216+ raise urllib.error.URLError(err)
217
218 for args in self.parent.addheaders:
219 h.putheader(*args)
220- for k, v in req.headers.items():
221+ for k, v in list(req.headers.items()):
222 h.putheader(k, v)
223 h.endheaders()
224 if req.data:
225@@ -336,7 +307,7 @@ def _start_transaction(self, h, req):
226 def _get_connection(self, host):
227 return NotImplementedError
228
229-class HTTPHandler(KeepAliveHandler, urllib2.HTTPHandler):
230+class HTTPHandler(KeepAliveHandler, urllib.request.HTTPHandler):
231 def __init__(self):
232 KeepAliveHandler.__init__(self)
233
234@@ -346,7 +317,7 @@ def http_open(self, req):
235 def _get_connection(self, host):
236 return HTTPConnection(host)
237
238-class HTTPSHandler(KeepAliveHandler, urllib2.HTTPSHandler):
239+class HTTPSHandler(KeepAliveHandler, urllib.request.HTTPSHandler):
240 def __init__(self, ssl_factory=None):
241 KeepAliveHandler.__init__(self)
242 if not ssl_factory:
243@@ -364,7 +335,7 @@ def _get_connection(self, host):
244 try: return self._ssl_factory.get_https_connection(host)
245 except AttributeError: return HTTPSConnection(host)
246
247-class HTTPResponse(httplib.HTTPResponse):
248+class HTTPResponse(http.client.HTTPResponse):
249 # we need to subclass HTTPResponse in order to
250 # 1) add readline() and readlines() methods
251 # 2) add close_connection() methods
252@@ -385,10 +356,7 @@ class HTTPResponse(httplib.HTTPResponse):
253
254
255 def __init__(self, sock, debuglevel=0, strict=0, method=None):
256- if method: # the httplib in python 2.3 uses the method arg
257- httplib.HTTPResponse.__init__(self, sock, debuglevel, method)
258- else: # 2.2 doesn't
259- httplib.HTTPResponse.__init__(self, sock, debuglevel)
260+ http.client.HTTPResponse.__init__(self, sock, debuglevel, method)
261 self.fileno = sock.fileno
262 self.code = None
263 self._rbuf = b""
264@@ -398,7 +366,7 @@ def __init__(self, sock, debuglevel=0, strict=0, method=None):
265 self._url = None # (same)
266 self._connection = None # (same)
267
268- _raw_read = httplib.HTTPResponse.read
269+ _raw_read = http.client.HTTPResponse.read
270
271 def close(self):
272 if self.fp:
273@@ -462,11 +430,11 @@ def readlines(self, sizehint = 0):
274 return list
275
276
277-class HTTPConnection(httplib.HTTPConnection):
278+class HTTPConnection(http.client.HTTPConnection):
279 # use the modified response class
280 response_class = HTTPResponse
281
282-class HTTPSConnection(httplib.HTTPSConnection):
283+class HTTPSConnection(http.client.HTTPSConnection):
284 response_class = HTTPResponse
285
286 #########################################################################
287@@ -474,81 +442,77 @@ class HTTPSConnection(httplib.HTTPSConnection):
288 #########################################################################
289
290 def error_handler(url):
291- global HANDLE_ERRORS
292- orig = HANDLE_ERRORS
293 keepalive_handler = HTTPHandler()
294- opener = urllib2.build_opener(keepalive_handler)
295- urllib2.install_opener(opener)
296- pos = {0: 'off', 1: 'on'}
297- for i in (0, 1):
298- print " fancy error handling %s (HANDLE_ERRORS = %i)" % (pos[i], i)
299- HANDLE_ERRORS = i
300- try:
301- fo = urllib2.urlopen(url)
302- foo = fo.read()
303- fo.close()
304- try: status, reason = fo.status, fo.reason
305- except AttributeError: status, reason = None, None
306- except IOError, e:
307- print " EXCEPTION: %s" % e
308- raise
309- else:
310- print " status = %s, reason = %s" % (status, reason)
311- HANDLE_ERRORS = orig
312+ opener = urllib.request.build_opener(keepalive_handler)
313+ urllib.request.install_opener(opener)
314+ try:
315+ fo = urllib.request.urlopen(url)
316+ foo = fo.read()
317+ fo.close()
318+ try: status, reason = fo.status, fo.reason
319+ except AttributeError: status, reason = None, None
320+ except IOError as e:
321+ print(" EXCEPTION: %s" % e)
322+ raise
323+ else:
324+ print(" status = %s, reason = %s" % (status, reason))
325 hosts = keepalive_handler.open_connections()
326- print "open connections:", hosts
327+ print("open connections:", hosts)
328 keepalive_handler.close_all()
329
330 def continuity(url):
331- import md5
332+ import hashlib
333 format = '%25s: %s'
334
335 # first fetch the file with the normal http handler
336- opener = urllib2.build_opener()
337- urllib2.install_opener(opener)
338- fo = urllib2.urlopen(url)
339+ opener = urllib.request.build_opener()
340+ urllib.request.install_opener(opener)
341+ fo = urllib.request.urlopen(url)
342 foo = fo.read()
343 fo.close()
344- m = md5.new(foo)
345- print format % ('normal urllib', m.hexdigest())
346+ m = hashlib.md5()
347+ m.update(foo)
348+ print(format % ('normal urllib', m.hexdigest()))
349
350 # now install the keepalive handler and try again
351- opener = urllib2.build_opener(HTTPHandler())
352- urllib2.install_opener(opener)
353+ opener = urllib.request.build_opener(HTTPHandler())
354+ urllib.request.install_opener(opener)
355
356- fo = urllib2.urlopen(url)
357+ fo = urllib.request.urlopen(url)
358 foo = fo.read()
359 fo.close()
360- m = md5.new(foo)
361- print format % ('keepalive read', m.hexdigest())
362+ m = hashlib.md5()
363+ m.update(foo)
364+ print(format % ('keepalive read', m.hexdigest()))
365
366- fo = urllib2.urlopen(url)
367- foo = ''
368+ fo = urllib.request.urlopen(url)
369+ foo = b''
370 while 1:
371 f = fo.readline()
372 if f: foo = foo + f
373 else: break
374 fo.close()
375- m = md5.new(foo)
376- print format % ('keepalive readline', m.hexdigest())
377+ m = hashlib.md5()
378+ m.update(foo)
379+ print(format % ('keepalive readline', m.hexdigest()))
380
381 def comp(N, url):
382- print ' making %i connections to:\n %s' % (N, url)
383+ print(' making %i connections to:\n %s' % (N, url))
384
385 sys.stdout.write(' first using the normal urllib handlers')
386 # first use normal opener
387- opener = urllib2.build_opener()
388- urllib2.install_opener(opener)
389+ opener = urllib.request.build_opener()
390+ urllib.request.install_opener(opener)
391 t1 = fetch(N, url)
392- print ' TIME: %.3f s' % t1
393+ print(' TIME: %.3f s' % t1)
394
395 sys.stdout.write(' now using the keepalive handler ')
396 # now install the keepalive handler and try again
397- opener = urllib2.build_opener(HTTPHandler())
398- urllib2.install_opener(opener)
399+ opener = urllib.request.build_opener(HTTPHandler())
400+ urllib.request.install_opener(opener)
401 t2 = fetch(N, url)
402- print ' TIME: %.3f s' % t2
403- print ' improvement factor: %.2f' % (t1/t2, )
404+ print(' TIME: %.3f s' % t2)
405+ print(' improvement factor: %.2f' % (t1/t2, ))
406
407 def fetch(N, url, delay=0):
408 import time
409@@ -556,7 +520,7 @@ def fetch(N, url, delay=0):
410 starttime = time.time()
411 for i in range(N):
412 if delay and i > 0: time.sleep(delay)
413- fo = urllib2.urlopen(url)
414+ fo = urllib.request.urlopen(url)
415 foo = fo.read()
416 fo.close()
417 lens.append(len(foo))
418@@ -566,7 +530,7 @@ def fetch(N, url, delay=0):
419 for i in lens[1:]:
420 j = j + 1
421 if not i == lens[0]:
422- print "WARNING: inconsistent length on read %i: %i" % (j, i)
423+ print("WARNING: inconsistent length on read %i: %i" % (j, i))
424
425 return diff
426
427@@ -574,16 +538,16 @@ def test_timeout(url):
428 global DEBUG
429 dbbackup = DEBUG
430 class FakeLogger:
431- def debug(self, msg, *args): print msg % args
432+ def debug(self, msg, *args): print(msg % args)
433 info = warning = error = debug
434 DEBUG = FakeLogger()
435- print " fetching the file to establish a connection"
436- fo = urllib2.urlopen(url)
437+ print(" fetching the file to establish a connection")
438+ fo = urllib.request.urlopen(url)
439 data1 = fo.read()
440 fo.close()
441
442 i = 20
443- print " waiting %i seconds for the server to close the connection" % i
444+ print(" waiting %i seconds for the server to close the connection" % i)
445 while i > 0:
446 sys.stdout.write('\r %2i' % i)
447 sys.stdout.flush()
448@@ -591,33 +555,33 @@ def debug(self, msg, *args): print msg % args
449 i -= 1
450 sys.stderr.write('\r')
451
452- print " fetching the file a second time"
453- fo = urllib2.urlopen(url)
454+ print(" fetching the file a second time")
455+ fo = urllib.request.urlopen(url)
456 data2 = fo.read()
457 fo.close()
458
459 if data1 == data2:
460- print ' data are identical'
461+ print(' data are identical')
462 else:
463- print ' ERROR: DATA DIFFER'
464+ print(' ERROR: DATA DIFFER')
465
466 DEBUG = dbbackup
467
468
469 def test(url, N=10):
470- print "checking error hander (do this on a non-200)"
471+ print("checking error hander (do this on a non-200)")
472 try: error_handler(url)
473- except IOError, e:
474- print "exiting - exception will prevent further tests"
475+ except IOError as e:
476+ print("exiting - exception will prevent further tests")
477 sys.exit()
478- print
479- print "performing continuity test (making sure stuff isn't corrupted)"
480+ print()
481+ print("performing continuity test (making sure stuff isn't corrupted)")
482 continuity(url)
483- print
484- print "performing speed comparison"
485+ print()
486+ print("performing speed comparison")
487 comp(N, url)
488- print
489- print "performing dropped-connection check"
490+ print()
491+ print("performing dropped-connection check")
492 test_timeout(url)
493
494 if __name__ == '__main__':
495@@ -627,7 +591,7 @@ def test(url, N=10):
496 N = int(sys.argv[1])
497 url = sys.argv[2]
498 except:
499- print "%s <integer> <url>" % sys.argv[0]
500+ print("%s <integer> <url>" % sys.argv[0])
501 else:
502 test(url, N)
503
504diff --git a/setup.py b/setup.py
505index 4f06360..0f963c1 100644
506--- a/setup.py
507+++ b/setup.py
508@@ -23,23 +23,13 @@
509 from setuptools import setup
510 import sys
511
512-try:
513- import six
514- py3 = six.PY3
515-except:
516- py3 = sys.version_info[0] >= 3
517-
518 # metadata
519-if py3:
520- import re
521- _version_re = re.compile(r'__version__\s*=\s*"(.*)"')
522- for line in open('keepalive/__init__.py', encoding='utf-8'):
523- version_match = _version_re.match(line)
524- if version_match:
525- _version = version_match.group(1)
526-else:
527- import keepalive
528- _version = keepalive.__version__
529+import re
530+_version_re = re.compile(r'__version__\s*=\s*"(.*)"')
531+for line in open('keepalive/__init__.py', encoding='utf-8'):
532+ version_match = _version_re.match(line)
533+ if version_match:
534+ _version = version_match.group(1)
535
536 setup(
537 name = 'keepalive',
538@@ -62,14 +52,10 @@
539 'Topic :: Software Development :: Libraries :: Python Modules',
540 'License :: OSI Approved :: GNU Lesser General Public License v2 or later (LGPLv2+)',
541 'Operating System :: OS Independent',
542- 'Programming Language :: Python :: 2',
543- 'Programming Language :: Python :: 2.6',
544- 'Programming Language :: Python :: 2.7',
545 'Programming Language :: Python :: 3',
546 'Programming Language :: Python :: 3.3',
547 'Programming Language :: Python :: 3.4',
548 'Programming Language :: Python :: 3.5',
549 ],
550- keywords = 'python http urllib keepalive',
551- use_2to3 = True
552+ keywords = 'python http urllib keepalive'
553 )