Skip to content

Commit 959400e

Browse files
author
littlelory
committed
改写httpClient
1 parent b4a4a67 commit 959400e

2 files changed

Lines changed: 97 additions & 76 deletions

File tree

‎java/HttpClient.java‎

Lines changed: 85 additions & 71 deletions
Original file line numberDiff line numberDiff line change
@@ -7,10 +7,8 @@
77
import org.apache.http.client.HttpResponseException;
88
import org.apache.http.client.ResponseHandler;
99
import org.apache.http.client.config.RequestConfig;
10-
import org.apache.http.client.methods.HttpGet;
11-
import org.apache.http.client.methods.HttpPost;
10+
import org.apache.http.client.methods.*;
1211
import org.apache.http.conn.ConnectionPoolTimeoutException;
13-
import org.apache.http.conn.HttpClientConnectionManager;
1412
import org.apache.http.entity.ContentType;
1513
import org.apache.http.entity.StringEntity;
1614
import org.apache.http.impl.client.DefaultHttpRequestRetryHandler;
@@ -27,7 +25,7 @@
2725
/**
2826
* HttpClient实体类
2927
* 这个类是对Apache的org.apache.http.client.HttpClient的一层封装
30-
*
28+
* <p>
3129
* Created by littlelory on 02/11/2017.
3230
*/
3331
public class HttpClient {
@@ -45,115 +43,131 @@ private HttpClient(org.apache.http.client.HttpClient httpClient, String key) {
4543

4644
/**
4745
* post请求
48-
* 向url参数传入的连接发送请求，请求的Body内容由body参数决定
46+
* 向url参数传入的连接发送post请求，请求的Body内容由body参数决定
4947
*
50-
* @param url 请求url
48+
* @param url 请求url
5149
* @param headerMap http header数据，key为header名，value为header值
52-
* @param body 请求的Body内容
50+
* @param body 请求的Body内容
5351
* @return 响应数据
5452
* @throws IOException 请求过程中发生异常
5553
*/
5654
public String doPost(String url, Map<String, String> headerMap, String body) throws IOException {
57-
log.info("[httpclient]["+key+"] to post, url = [" + url + "], headerMap = [" + headerMap + "], body = [" + body + "].");
58-
59-
//创建请求实例
55+
log.debug("[httpclient][" + key + "] to post, url = [" + url + "], headerMap = [" + headerMap + "], body = [" + body + "].");
6056
HttpPost request = new HttpPost(url);
61-
62-
//设置Headers
63-
if (headerMap != null && headerMap.size() > 0) {
64-
int headerSize = headerMap.size();
65-
Header[] headers = new Header[headerSize];
66-
int index = 0;
67-
for (Map.Entry<String, String> entry : headerMap.entrySet()) {
68-
headers[index] = new BasicHeader(entry.getKey(), entry.getValue());
69-
index++;
70-
}
71-
request.setHeaders(headers);
72-
}
73-
74-
//设置request body
75-
StringEntity entity = new StringEntity(body);
76-
request.setEntity(entity);
77-
78-
//发起请求
79-
try {
80-
long start = System.currentTimeMillis();
81-
String response = httpClient.execute(request, responseHandler);
82-
long cost = System.currentTimeMillis() - start;
83-
log.info(String.format("[httpclient]["+key+"][cost] url[%s], postMethod cost time:%dms", url, cost));
84-
return response;
85-
} catch (ConnectionPoolTimeoutException e) {
86-
log.warn("[httpcient]["+key+"][timeout] connection pool timeout, url[" + url + "], msg[" + e.getMessage() + "].");
87-
throw e;
88-
} catch (SocketTimeoutException e) {
89-
log.warn("[httpcient]["+key+"][timeout] socket timeout, url[" + url + "], msg[" + e.getMessage() + "].");
90-
throw e;
91-
} catch (IOException e) {
92-
log.error(String.format("[httpclient]["+key+"][error] url[%s], msg = [%s].", url, e.getMessage()), e);
93-
throw e;
94-
}
57+
return doHttpEntityEnclosingRequestBase(request, headerMap, body);
9558
}
96-
59+
9760
/**
9861
* get请求
9962
* 发送get请求，请求的url会根据url参数和params参数共同决定
10063
* 例如url参数="www.xxx.com",params参数="uid=1&timestamp=1510557964"，
10164
* 则最终的请求url="www.xxx.com?uid=1&timestamp=1510557964"
10265
*
103-
* @param url 请求url
66+
* @param url 请求url
10467
* @param headerMap http header数据，key为header名，value为header值
105-
* @param params 请求参数
68+
* @param params 请求参数
10669
* @return 响应数据
10770
* @throws IOException 请求过程中发生异常
10871
*/
10972
public String doGet(String url, Map<String, String> headerMap, String params) throws IOException {
110-
log.info("[httpclient]["+key+"] to get, url = [" + url + "], headerMap = [" + headerMap + "], params = [" + params + "].");
73+
log.debug("[httpclient][" + key + "] to get, url = [" + url + "], headerMap = [" + headerMap + "], params = [" + params + "].");
11174
//创建请求实例
11275
HttpGet request = new HttpGet(url + (params != null && params.length() > 0 ? "?" + params : ""));
76+
return doHttpRequestBase(request, headerMap);
77+
}
11378

114-
if (headerMap != null && headerMap.size() > 0) {
115-
//设置Headers
116-
int headerSize = headerMap.size();
117-
Header[] headers = new Header[headerSize];
118-
int index = 0;
119-
for (Map.Entry<String, String> entry : headerMap.entrySet()) {
120-
headers[index] = new BasicHeader(entry.getKey(), entry.getValue());
121-
index++;
122-
}
123-
request.setHeaders(headers);
79+
/**
80+
* delete请求
81+
*
82+
* @param url 请求url
83+
* @param headerMap http header数据，key为header名，value为header值
84+
* @return 响应数据
85+
* @throws IOException 请求过程中发生异常
86+
*/
87+
public String doDelete(String url, Map<String, String> headerMap) throws IOException {
88+
log.debug("[httpclient][" + key + "] to delete, url = [" + url + "], headerMap = [%s], params = [" + headerMap + "].");
89+
HttpDelete request = new HttpDelete(url);
90+
return doHttpRequestBase(request, headerMap);
91+
}
92+
93+
/**
94+
* 执行Get、Delete请求
95+
*
96+
* @param request 请求实例
97+
* @param headerMap 请求头数据
98+
* @return 响应数据
99+
* @throws ConnectionPoolTimeoutException 如果从connectionManager获取client的过程超时，会出现本异常
100+
* @throws SocketTimeoutException 如果请求超时，会出现本异常
101+
* @throws IOException 请求过程出现异常
102+
*/
103+
private String doHttpRequestBase(HttpRequestBase request, Map<String, String> headerMap) throws IOException {
104+
setHeaders(request, headerMap);
105+
return doRequest(request, responseHandler);
106+
}
107+
108+
/**
109+
* 执行Post、Put、Patch请求
110+
* 由于Post、Put、Patch三个method会有消息体，所以httpclient对这三个单独抽象，都属于org.apache.http.client.methods.HttpEntityEnclosingRequestBase的子类
111+
*
112+
* @param request 请求实例
113+
* @param headerMap 请求头数据
114+
* @param body 消息体信息
115+
* @return 响应数据
116+
* @throws ConnectionPoolTimeoutException 如果从connectionManager获取client的过程超时，会出现本异常
117+
* @throws SocketTimeoutException 如果请求超时，会出现本异常
118+
* @throws IOException 请求过程出现异常
119+
*/
120+
private String doHttpEntityEnclosingRequestBase(HttpEntityEnclosingRequestBase request, Map<String, String> headerMap, String body) throws IOException {
121+
setHeaders(request, headerMap);
122+
if (body != null && body.length() > 0) {
123+
request.setEntity(new StringEntity(body));
124124
}
125+
return doRequest(request, responseHandler);
126+
}
127+
128+
private void setHeaders(HttpRequestBase request, Map<String, String> headerMap) {
129+
if (headerMap == null || headerMap.size() == 0)
130+
return;
131+
132+
int headerSize = headerMap.size();
133+
Header[] headers = new Header[headerSize];
134+
int index = 0;
135+
for (Map.Entry<String, String> entry : headerMap.entrySet()) {
136+
headers[index] = new BasicHeader(entry.getKey(), entry.getValue());
137+
index++;
138+
}
139+
request.setHeaders(headers);
140+
}
141+
142+
private String doRequest(HttpRequestBase request, ResponseHandler<String> handler) throws IOException {
125143

126-
//发起请求
127144
try {
128145
long start = System.currentTimeMillis();
129-
String response = httpClient.execute(request, responseHandler);
146+
String response = httpClient.execute(request, handler);
130147
long cost = System.currentTimeMillis() - start;
131-
log.info(String.format("[httpclient]["+key+"][cost] url[%s], postMethod cost time:%dms", url, cost));
148+
log.debug("[httpclient][" + key + "][cost] request[" + request + "], postMethod cost time:" + cost + "ms");
132149
return response;
133150
} catch (ConnectionPoolTimeoutException e) {
134-
log.warn("[httpcient]["+key+"][timeout] connection pool timeout, url[" + url + "], msg[" + e.getMessage() + "].");
151+
log.warn("[httpcient][" + key + "][timeout] connection pool timeout, request[" + request + "], msg[" + e.getMessage() + "].");
135152
throw e;
136153
} catch (SocketTimeoutException e) {
137-
log.warn("[httpcient]["+key+"][timeout] socket timeout, url[" + url + "], msg[" + e.getMessage() + "].");
154+
log.warn("[httpcient][" + key + "][timeout] socket timeout, request[" + request + "], msg[" + e.getMessage() + "].");
138155
throw e;
139156
} catch (IOException e) {
140-
log.error(String.format("[httpclient]["+key+"][error] url[%s], msg = [%s].", url, e.getMessage()), e);
157+
log.error("[httpclient][" + key + "][error] url[%s], msg = [" + e.getMessage() + "].", e);
141158
throw e;
142159
}
143160
}
144161

145162
//请求响应Handler
146163
private ResponseHandler<String> responseHandler = new ResponseHandler<String>() {
147164
@Override
148-
public String handleResponse(
149-
final HttpResponse response) throws IOException {
165+
public String handleResponse(final HttpResponse response) throws IOException {
150166
StatusLine statusLine = response.getStatusLine();
151167
HttpEntity entity = response.getEntity();
152-
log.info("[httpclient]["+key+"] response: statusLine[" + statusLine + "].");
168+
log.debug("[httpclient][" + key + "] response: statusLine[" + statusLine + "].");
153169
if (statusLine.getStatusCode() != HttpStatus.SC_OK) {
154-
throw new HttpResponseException(
155-
statusLine.getStatusCode(),
156-
statusLine.getReasonPhrase());
170+
throw new HttpResponseException(statusLine.getStatusCode(), statusLine.getReasonPhrase());
157171
}
158172
if (entity == null) {
159173
throw new ClientProtocolException("Response contains no content.");
@@ -173,7 +187,7 @@ public String handleResponse(
173187
* 创建HttpClient的Builder实例
174188
*
175189
* @param connectionManager 管理此client的HttpClientManager
176-
* @param key 此client的标识
190+
* @param key 此client的标识
177191
* @return Builder实例
178192
*/
179193
public static Builder instance(HttpClientManager connectionManager, String key) {

‎python/blog_monitor/jianshu_monitor.py‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,22 @@
1818
reload(sys)
1919
sys.setdefaultencoding('utf-8')
2020

21+
if len(sys.argv) != 2:
22+
print('usage: python jianshu_monitor.py base_path')
23+
exit(15)
24+
25+
base_path = sys.argv[1]
26+
2127
logger = logging.getLogger('monitor')
2228
formatter = logging.Formatter('%(asctime)s %(levelname)-8s: %(message)s')
2329

24-
file_handler = logging.FileHandler('run.log')
30+
file_handler = logging.FileHandler('%s/run.log' % base_path)
2531
file_handler.setFormatter(formatter)
2632
logger.addHandler(file_handler)
2733

28-
# console_handler = logging.StreamHandler(sys.stdout)
29-
# console_handler.setFormatter(formatter)
30-
# logger.addHandler(console_handler)
34+
console_handler = logging.StreamHandler(sys.stdout)
35+
console_handler.setFormatter(formatter)
36+
logger.addHandler(console_handler)
3137

3238
logger.setLevel(logging.INFO)
3339

@@ -94,7 +100,7 @@ def main():
94100
target_url = 'https://www.jianshu.com/u/2d48ed845229'
95101
try:
96102
i = 1
97-
f = codecs.open('page_info.log', 'a+')
103+
f = codecs.open('%s/page_info.log' % base_path, 'a+')
98104
while True:
99105
url = '%s?order_by=shared_at&page=%d' % (target_url, i)
100106
code, body = http_request(url)
@@ -115,3 +121,4 @@ def main():
115121

116122
if __name__ == '__main__':
117123
main()
124+

0 commit comments

Comments
 (0)