-
Notifications
You must be signed in to change notification settings - Fork 26
Expand file tree
/
Copy pathhttp_layer.py
More file actions
189 lines (153 loc) · 6.44 KB
/
http_layer.py
File metadata and controls
189 lines (153 loc) · 6.44 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
import json
from .sword2_logging import logging
http_l = logging.getLogger(__name__)
class HttpResponse(object):
def __init__(self, *args, **kwargs):
pass
def __getitem__(self, att):
# needs to behave like a dictionary
# we need to be able to look up at least:
# content-type
# status (as an integer)
# location
pass
def __repr__(self):
return json.dumps(self.__dict__, indent=True)
def get(self, att, default=None):
# same as __getattr__ but with default return
pass
def keys(self):
pass
class HttpLayer(object):
def __init__(self, *args, **kwargs): pass
def add_credentials(self, username, password): pass
def request(self, uri, method, headers=None, payload=None):
# should return a tuple of an HttpResponse object and the content
pass
################################################################################
# Default httplib2 implementation
################################################################################
import httplib2
class HttpLib2Response(HttpResponse):
def __init__(self, response):
self.resp = response
self.status = int(self.resp.status)
def __getitem__(self, att):
if att == "status":
return self.status
return self.resp.get(att)
def get(self, att, default=None):
if att == "status":
return self.status
return self.resp.get(att, default)
def keys(self):
return list(self.resp.keys())
class HttpLib2Layer(HttpLayer):
def __init__(self, cache_dir=".cache", timeout=30.0, ca_certs=None):
self.h = httplib2.Http(cache_dir, timeout=timeout, ca_certs=ca_certs)
def add_credentials(self, username, password):
self.h.add_credentials(username, password)
def request(self, uri, method, headers=None, payload=None):
if hasattr(payload, 'read'):
# Need to work out why a 401 challenge will stop httplib2 from sending the file...
# likely need to make it re-seek to 0...
# FIXME: In the meantime, read the file into memory... *sigh*
payload = payload.read()
resp, content = self.h.request(uri, method, headers=headers, body=payload)
return (HttpLib2Response(resp), content)
################################################################################
# Guest urllib2 implementation
################################################################################
import urllib.request, urllib.error, urllib.parse, base64
class PreemptiveBasicAuthHandler(urllib.request.HTTPBasicAuthHandler):
def __init__(self, username, password):
self.username = username
self.password = password
def http_request(self, request):
request.add_header(self.auth_header, 'Basic %s' % base64.b64encode(self.username + ':' + self.password))
return request
https_request = http_request
class UrlLib2Response(HttpResponse):
def __init__(self, response):
self.response = response
self.headers = dict(response.info())
self.status = int(self.response.code)
def __getitem__(self, att):
# needs to behave like a dictionary
# we need to be able to look up at least:
# content-type
# status
# location
if att == "status":
return self.status
return self.headers[att]
def get(self, att, default=None):
# same as __getattr__ but with default return
if att == "status":
return self.status
return self.headers.get(att, default)
def keys(self):
return list(self.headers.keys()) + ["status"]
# http://stackoverflow.com/questions/2502596/python-http-post-a-large-file-with-streaming
"""
import urllib2
import mmap
# Open the file as a memory mapped string. Looks like a string, but
# actually accesses the file behind the scenes.
f = open('somelargefile.zip','rb')
mmapped_file_as_string = mmap.mmap(f.fileno(), 0, access=mmap.ACCESS_READ)
# Do the request
request = urllib2.Request(url, mmapped_file_as_string)
request.add_header("Content-Type", "application/zip")
response = urllib2.urlopen(request)
#close everything
mmapped_file_as_string.close()
f.close()
"""
class UrlLib2Layer(HttpLayer):
def __init__(self, opener=None):
self.opener = opener
if self.opener is None:
self.opener = urllib.request.build_opener()
def add_credentials(self, username, password):
auth_handler = PreemptiveBasicAuthHandler(username, password)
current_handlers = self.opener.handlers
new_handlers = current_handlers + [auth_handler]
self.opener = urllib.request.build_opener(*new_handlers)
def request(self, uri, method, headers=None, payload=None):
# NOTE: payload can be a file or a string
if headers is None:
headers = {}
# should return a tuple of an HttpResponse object and the content
try:
if method == "GET":
req = urllib.request.Request(uri, None, headers)
response = self.opener.open(req)
return UrlLib2Response(response), response.read()
elif method == "POST":
req = urllib.request.Request(uri, payload, headers)
response = self.opener.open(req)
return UrlLib2Response(response), response.read()
elif method == "PUT":
req = urllib.request.Request(uri, payload, headers)
# monkey-patch the request method (which seems to be the fastest
# way to do this)
req.get_method = lambda: 'PUT'
response = self.opener.open(req)
return UrlLib2Response(response), response.read()
elif method == "DELETE":
req = urllib.request.Request(uri, None, headers)
# monkey-patch the request method (which seems to be the fastest
# way to do this)
req.get_method = lambda: 'DELETE'
response = self.opener.open(req)
return UrlLib2Response(response), response.read()
else:
raise NotImplementedError()
except urllib.error.HTTPError as e:
try:
# treat it like a normal response
return UrlLib2Response(e), e.read()
except Exception as e:
# unable to read()
return UrlLib2Response(e), None