httpclient
httpclient是java里一大坑,出了问题比较难排查，很多时候是对参数不了解，随意设置导致
basic
线程实现
httpclient 3.1 用synchronized + wait + notifyAll实现，大量请求时，线程饥饿
4 采用ReentrantLock（默认非公平） + Condition（每个线程一个）实现，这种情况几乎不会出现
连接池
- 降低延迟：不用连接池，每次连接发起会重新建立TCP连接(3次握手)，用关闭连接(4次挥手)，基本上3倍延迟
- 支持更大的并发：不用连接池，每次连接打开一个端口，端口资源很快用完，无法建立新的连接
HttpRequestRetryHandler
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
|
if (executionCount >= 5) {
// Do not retry if over max retry count
return false;
}
if (exception instanceof InterruptedIOException) {
// Timeout
return false;
}
if (exception instanceof UnknownHostException) {
// Unknown host
return false;
}
if (exception instanceof ConnectTimeoutException) {
// Connection refused
return false;
}
if (exception instanceof SSLException) {
// SSL handshake exception
return false;
}
后面有一段
HttpClientContext clientContext = HttpClientContext.adapt(context);
HttpRequest request = clientContext.getRequest();
// 如果请求是幂等的，就再次尝试
if (!(request instanceof HttpEntityEnclosingRequest)) {
return true;
}
return false;
|
就是上面这段，默认 get,head 可重试，post不行，可以直接返回 true 进行retry
ConnectionManager
早期版本 PoolingClientConnectionManager,DefaultHttpClient 在4.3.x 后大部分已过时
最新的PoolingHttpClientConnectionManager
1
2
3
4
5
6
7
8
9
10
11
12
13
|
PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager();
cm.setMaxTotal(200);
// 将每个路由基础的连接
cm.setDefaultMaxPerRoute(20);
// 将目标主机的最大连接数增加到50
HttpHost localhost = new HttpHost("ddatsh.com", 80);
cm.setMaxPerRoute(new HttpRoute(localhost), 50);
CloseableHttpClient httpClient = HttpClients.custom()
.setConnectionManager(cm)
.build();
|
1
2
3
4
5
6
7
|
CloseableHttpClient httpclient = HttpClients.createDefault();
HttpHead httpHead = new HttpHead("https://ddatsh.com");
CloseableHttpResponse response = httpclient.execute(httpHead);
HeaderIterator it = response.headerIterator();
while (it.hasNext()) {
System.out.println(it.next());
}
|
完整连接示例
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
|
import org.apache.http.*;
import org.apache.http.client.*;
import org.apache.http.client.config.*;
import org.apache.http.client.methods.*;
import org.apache.http.client.protocol.*;
import org.apache.http.config.*;
import org.apache.http.conn.*;
import org.apache.http.conn.routing.*;
import org.apache.http.impl.client.*;
import org.apache.http.impl.conn.*;
import org.apache.http.protocol.*;
import javax.net.ssl.*;
import java.io.*;
import java.net.*;
import java.nio.charset.*;
public class PoolTest {
public static void main(String[] args) throws IOException {
/**
* 创建连接管理器，并设置相关参数
*/
//连接管理器，使用无惨构造
PoolingHttpClientConnectionManager connManager
= new PoolingHttpClientConnectionManager();
/**
* 连接数相关设置
*/
//最大连接数
connManager.setMaxTotal(200);
//默认的每个路由的最大连接数
connManager.setDefaultMaxPerRoute(100);
//设置到某个路由的最大连接数，会覆盖defaultMaxPerRoute
connManager.setMaxPerRoute(new HttpRoute(new HttpHost("somehost", 80)), 150);
/**
* socket配置（默认配置 和 某个host的配置）
*/
SocketConfig socketConfig = SocketConfig.custom()
.setTcpNoDelay(true) //是否立即发送数据，设置为true会关闭Socket缓冲，默认为false
.setSoReuseAddress(true) //是否可以在一个进程关闭Socket后，即使它还没有释放端口，其它进程还可以立即重用端口
.setSoTimeout(500) //接收数据的等待超时时间，单位ms
.setSoLinger(60) //关闭Socket时，要么发送完所有数据，要么等待60s后，就关闭连接，此时socket.close()是阻塞的
.setSoKeepAlive(true) //开启监视TCP连接是否有效
.build();
connManager.setDefaultSocketConfig(socketConfig);
connManager.setSocketConfig(new HttpHost("somehost", 80), socketConfig);
/**
* HTTP connection相关配置（默认配置 和 某个host的配置）
* 一般不修改HTTP connection相关配置，故不设置
*/
//消息约束
MessageConstraints messageConstraints = MessageConstraints.custom()
.setMaxHeaderCount(200)
.setMaxLineLength(2000)
.build();
//Http connection相关配置
ConnectionConfig connectionConfig = ConnectionConfig.custom()
.setMalformedInputAction(CodingErrorAction.IGNORE)
.setUnmappableInputAction(CodingErrorAction.IGNORE)
.setCharset(Consts.UTF_8)
.setMessageConstraints(messageConstraints)
.build();
//一般不修改HTTP connection相关配置，故不设置
//connManager.setDefaultConnectionConfig(connectionConfig);
//connManager.setConnectionConfig(new HttpHost("somehost", 80), ConnectionConfig.DEFAULT);
/**
* request请求相关配置
*/
RequestConfig defaultRequestConfig = RequestConfig.custom()
.setConnectTimeout(2 * 1000) //连接超时时间
.setSocketTimeout(2 * 1000) //读超时时间（等待数据超时时间）
.setConnectionRequestTimeout(500) //从池中获取连接超时时间
.setStaleConnectionCheckEnabled(true)//检查是否为陈旧的连接，默认为true，类似testOnBorrow
.build();
/**
* 重试处理
* 默认是重试3次
*/
//禁用重试(参数：retryCount、requestSentRetryEnabled)
HttpRequestRetryHandler requestRetryHandler = new DefaultHttpRequestRetryHandler(0, false);
//自定义重试策略
HttpRequestRetryHandler myRetryHandler = new HttpRequestRetryHandler() {
public boolean retryRequest(IOException exception, int executionCount, HttpContext context) {
//Do not retry if over max retry count
if (executionCount >= 3) {
return false;
}
//Timeout
if (exception instanceof InterruptedIOException) {
return false;
}
//Unknown host
if (exception instanceof UnknownHostException) {
return false;
}
//Connection refused
if (exception instanceof ConnectTimeoutException) {
return false;
}
//SSL handshake exception
if (exception instanceof SSLException) {
return false;
}
HttpClientContext clientContext = HttpClientContext.adapt(context);
HttpRequest request = clientContext.getRequest();
boolean idempotent = !(request instanceof HttpEntityEnclosingRequest);
//Retry if the request is considered idempotent
//如果请求类型不是HttpEntityEnclosingRequest，被认为是幂等的，那么就重试
//HttpEntityEnclosingRequest指的是有请求体的request，比HttpRequest多一个Entity属性
//而常用的GET请求是没有请求体的，POST、PUT都是有请求体的
//Rest一般用GET请求获取数据，故幂等，POST用于新增数据，故不幂等
if (idempotent) {
return true;
}
return false;
}
};
/**
* 创建httpClient
*/
CloseableHttpClient httpclient = HttpClients.custom()
.setConnectionManager(connManager) //连接管理器
.setProxy(new HttpHost("myproxy", 8080)) //设置代理
.setDefaultRequestConfig(defaultRequestConfig) //默认请求配置
.setRetryHandler(myRetryHandler) //重试策略
.build();
//创建一个Get请求，并重新设置请求参数，覆盖默认
HttpGet httpget = new HttpGet("http://www.somehost.com/");
RequestConfig requestConfig = RequestConfig.copy(defaultRequestConfig)
.setSocketTimeout(5000)
.setConnectTimeout(5000)
.setConnectionRequestTimeout(5000)
.setProxy(new HttpHost("myotherproxy", 8080))
.build();
httpget.setConfig(requestConfig);
CloseableHttpResponse response = null;
try {
//执行请求
response = httpclient.execute(httpget);
HttpEntity entity = response.getEntity();
// If the response does not enclose an entity, there is no need
// to bother about connection release
if (entity != null) {
InputStream instream = entity.getContent();
try {
instream.read();
// do something useful with the response
} catch (IOException ex) {
// In case of an IOException the connection will be released
// back to the connection manager automatically
throw ex;
} finally {
// Closing the input stream will trigger connection release
// 释放连接回到连接池
instream.close();
}
}
} catch (Exception e) {
e.printStackTrace();
} finally {
if (response != null) {
try {
//关闭连接(如果已经释放连接回连接池，则什么也不做)
response.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (httpclient != null) {
try {
//关闭连接管理器，并会关闭其管理的连接
httpclient.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
}
}
|
fluent-hc
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
|
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import java.io.BufferedReader;
import java.io.InputStreamReader;
public class Main {
public static void main(String[] args) throws Exception {
CloseableHttpClient httpclient = HttpClients.createDefault();
HttpGet httpGet = new HttpGet("https://ddatsh.com");
CloseableHttpResponse response1 = httpclient.execute(httpGet);
StringBuilder result = new StringBuilder();
try {
BufferedReader rd = new BufferedReader(
new InputStreamReader(response1.getEntity().getContent()));
String line;
while ((line = rd.readLine()) != null) {
result.append(line);
}
} catch (Exception e) {
e.printStackTrace();
} finally {
response1.close();
}
System.out.println(result.toString());
}
}
|
了fluent之后
1
2
3
4
5
6
7
|
import org.apache.http.client.fluent.Request;
public class Main {
public static void main(String[] args) throws Exception {
System.out.println(Request.Get("https://ddatsh.com").execute().returnContent().toString());
}
}
|
代码简洁了很多，而且打印出来的结果也是按照标准的html样式格式化了的， 而不是像上面的那个一样全部代码都在一行，内容也是转码了的
ref
HttpClient 4.3连接池参数配置及源码解读
categories:
dev
tags:
httpclient