我们公司以流量话费为主，今天下来一个贼奇葩的通道。天津移动的一家通道，需要对他们的管理系统进行抓包。然后上传一个txt文件作为充值的手机号码和面值。那么后台模拟登录和上传文件成了大问题。这里写一下调试心得，希望遇到的同行们能够少走弯路。
1.登录模块。
其实本网站的登录还是挺简单的，更让人开心的是它的验证码是你给它请求多少就给你返回相对应验证码图片。所以不需要借助第三方进行验证码识别。这里不多说直接上请求代码。
/**
* 使用get方式抓取网页内容
*
* @param url
* url地址
* @param defaultResponseCharset
* http默认响应字符集
* @param headers
* 请求头
* @return html代码
* @throws Exception
*/
public String getStringFromUrl(String url, String defaultResponseCharset,
Map<String, String> headers) throws Exception {
/*
* boolean handleRedirects = headers != null &&
* headers.containsKey(HANDLE_REDIRECTS);
*/
boolean handleRedirects = true;// 默认支持301/302跳转
CloseableHttpResponse response = null;
HttpClientContext context = this.context.get();
retry = 0;
try {
while (retry < retryCount) {
try {
HttpUriRequest get = createHttpGet(url, headers,
handleRedirects);
timeWatcher.watch();
response = client.execute(get, context);
StatusLine statusLine = response.getStatusLine();
int statusCode = statusLine.getStatusCode();
this.currentStatusCode.set(statusCode);
Header header = response.getFirstHeader("Content-Type");
this.contentType.set(header != null ? header.getValue()
: null);
this.headers.set(response.getAllHeaders());
header = response.getFirstHeader("Location");
this.location
.set(header != null ? header.getValue() : null);
HttpEntity entity = response.getEntity();
InputStream in = entity.getContent();
if (statusCode == HttpStatus.SC_NOT_FOUND) {
return null;
}
else if (statusCode == HttpStatus.SC_INTERNAL_SERVER_ERROR
|| statusCode == HttpStatus.SC_BAD_GATEWAY
|| statusCode == HttpStatus.SC_SERVICE_UNAVAILABLE
|| statusCode == HttpStatus.SC_GATEWAY_TIMEOUT
|| statusCode == HttpStatus.SC_FORBIDDEN) {
throw new Exception(statusLine.toString());
}
else if (!handleRedirects
&& (statusCode != HttpStatus.SC_OK
&& statusCode != HttpStatus.SC_PARTIAL_CONTENT && statusCode != HttpStatus.SC_NOT_MODIFIED)) {
throw new CommonLogicException(statusCode,
statusLine.toString());
}
else if (in == null) {
throw new Exception("抓取文件失败" + statusLine);
}
ContentType contentType = ContentType.get(entity);
Charset charset = null;
if (contentType != null) {
checkTextContentType(entity.getContentLength(),
contentType);
charset = contentType.getCharset();
}
if (charset == null) {
charset = Charset.forName(defaultResponseCharset);
}
return IOUtils.toString(in, charset);
}
catch (CommonLogicException ex) {
throw ex;
}
catch (Exception ex) {
if (response != null) {
try {
response.close();
response = null;
}
catch (Exception ex1) {
}
}
releaseConnection();
watchForError();
if (retry >= retryCount) {
throw ex;
}
}
finally {
if (response != null) {
try {
response.close();
}
catch (Exception ex) {
}
}
}
}
}
finally {
this.context.remove();
}
return null;
}2.后台发送文件及数据。
这里其实我一直纠结于设置multipart/form-data，其实不用那么麻烦。大家看下我抽取的代码就懂了。
/**
* 使用post方式抓取网页内容
*
* @param url
* url地址
* @param defaultResponseCharset
* http默认响应字符集
* @param parameters
* post参数
* @param postCharset
* post请求字符集
* @param headers
* 请求头
* @return html代码
* @throws Exception
*/
public String getStringFromUrl(String url, String defaultResponseCharset,
Map<String, Object> parameters, String postCharset,
Map<String, String> headers) throws Exception {
/*
* boolean handleRedirects = headers != null &&
* headers.containsKey(HANDLE_REDIRECTS);
*/
boolean handleRedirects = true;
CloseableHttpResponse response = null;
HttpClientContext context = this.context.get();
retry = 0;
try {
while (retry < retryCount) {
try {
HttpUriRequest post = createHttpPost(url, parameters,
postCharset, headers, handleRedirects);
timeWatcher.watch();
response = client.execute(post, context);
StatusLine statusLine = response.getStatusLine();
int statusCode = statusLine.getStatusCode();
this.currentStatusCode.set(statusCode);
Header header = response.getFirstHeader("Content-Type");
this.contentType.set(header != null ? header.getValue()
: null);
this.headers.set(response.getAllHeaders());
header = response.getFirstHeader("Location");
this.location
.set(header != null ? header.getValue() : null);
HttpEntity entity = response.getEntity();
InputStream in = entity.getContent();
if (statusCode == HttpStatus.SC_NOT_FOUND) {
return null;
}
else if (statusCode == HttpStatus.SC_INTERNAL_SERVER_ERROR
|| statusCode == HttpStatus.SC_BAD_GATEWAY
|| statusCode == HttpStatus.SC_SERVICE_UNAVAILABLE
|| statusCode == HttpStatus.SC_GATEWAY_TIMEOUT
|| statusCode == HttpStatus.SC_FORBIDDEN) {
throw new Exception(statusLine.toString());
}
else if (!handleRedirects
&& (statusCode != HttpStatus.SC_OK
&& statusCode != HttpStatus.SC_PARTIAL_CONTENT && statusCode != HttpStatus.SC_NOT_MODIFIED)) {
throw new CommonLogicException(statusCode,
statusLine.toString());
}
else if (in == null) {
throw new Exception("抓取文件失败" + statusLine);
}
ContentType contentType = ContentType.get(entity);
Charset charset = null;
if (contentType != null) {
checkTextContentType(entity.getContentLength(),
contentType);
charset = contentType.getCharset();
}
if (charset == null) {
charset = Charset.forName(defaultResponseCharset);
}
return IOUtils.toString(in, charset);
}
catch (CommonLogicException ex) {
throw ex;
}
catch (Exception ex) {
if (response != null) {
try {
response.close();
response = null;
}
catch (Exception ex1) {
}
}
releaseConnection();
watchForError();
if (retry >= retryCount) {
throw ex;
}
}
finally {
if (response != null) {
try {
response.close();
}
catch (Exception ex) {
}
}
}
}
}
finally {
this.context.remove();
}
return null;
}这里面有个createPost方法，我给大家写下。
/**
* 生成HttpPost请求
*
* @param url
* url地址
* @param parameters
* post参数
* @param postCharset
* post请求字符集
* @param headers
* 请求头
* @param handleRedirects
* 是否支持跳转
* @return HttpPost
*/
private HttpUriRequest createHttpPost(String url,
Map<String, Object> parameters, String postCharset,
Map<String, String> headers, boolean handleRedirects)
throws Exception {
HttpClientContext context = this.context.get();
RequestConfig.Builder configBuilder = RequestConfig.custom();
configBuilder.setConnectTimeout(connectionTimeout);
configBuilder.setSocketTimeout(soTimeout);
configBuilder.setConnectionRequestTimeout(connectionTimeout);
configBuilder.setMaxRedirects(5);
configBuilder.setCookieSpec(cookieSpecs);
if (keepSession && cookieStore != null) {
context.setCookieStore(cookieStore);
}
if (proxyTask != null) {
httpProxy = proxyTask.selectRandomProxy();
}
if (httpProxy != null) {
HttpHost proxy = new HttpHost(httpProxy.ip, httpProxy.port);
configBuilder.setProxy(proxy);
if (StringUtils.isNotBlank(httpProxy.user)) {
CredentialsProvider credsProvider = new BasicCredentialsProvider();
credsProvider.setCredentials(AuthScope.ANY,
new UsernamePasswordCredentials(httpProxy.user,
httpProxy.password));
context.setCredentialsProvider(credsProvider);
}
}
if (handleRedirects) {
configBuilder.setRedirectsEnabled(true);
configBuilder.setRelativeRedirectsAllowed(true);
configBuilder.setMaxRedirects(8);
}
else {
configBuilder.setRedirectsEnabled(false);
}
configBuilder.setAuthenticationEnabled(authenticationEnabled);
RequestBuilder reqBuilder = RequestBuilder.post();
reqBuilder.setConfig(configBuilder.build());
reqBuilder.setUri(url.trim());
reqBuilder.setHeader("User-Agent", CList.USER_AGENT_FIREFOX);
reqBuilder.setHeader("Accept", "*/*");
reqBuilder.setHeader("Connection", "keep-alive");
if (useringXForwardedFor) {
reqBuilder.setHeader(CList.HTTP_X_FORWARDED_FOR,
ProxyTask.randomXForwardedFor());
}
if (headers != null && !headers.isEmpty()) {
for (Entry<String, String> entry : headers.entrySet()) {
String name = entry.getKey();
if (StringUtils.isBlank(name) || HANDLE_REDIRECTS.equals(name)
|| ABORT_REQUEST.equals(name)) {
continue;
}
String value = entry.getValue();
if (StringUtils.isBlank(value)) {
reqBuilder.removeHeaders(name);
}
else {
reqBuilder.setHeader(name, value);
}
}
}
if (parameters != null && !parameters.isEmpty()) {
// 0:普通类型 1:multipart类型 2:自定义类型 3:json
int bodyType = 0;
if (parameters.containsKey("json")) {
bodyType = 3;
} else {
for (Object value : parameters.values()) {
if (value instanceof File || value instanceof ContentBody) {
bodyType = 1;
break;
} else if (value.getClass().isArray()) {
for (int i = 0; i < Array.getLength(value); i++) {
Object v = Array.get(value, i);
if (v instanceof File || v instanceof ContentBody) {
bodyType = 1;
break;
}
}
if (bodyType == 1) {
break;
}
} else if (value instanceof HttpEntity) {
bodyType = 2;
break;
}
}
}
if (bodyType == 1) {
MultipartEntityBuilder meBuilder = MultipartEntityBuilder
.create();
for (Entry<String, Object> entry : parameters.entrySet()) {
String key = entry.getKey();
Object value = entry.getValue();
if (key == null || value == null) {
continue;
}
if (value instanceof File) {
File f = (File) value;
meBuilder.addBinaryBody(key, f,
ContentType.APPLICATION_OCTET_STREAM,
f.getName());
}
else if (value instanceof ContentBody) {
meBuilder.addPart(key, (ContentBody) value);
}
else if (value.getClass().isArray()) {
for (int i = 0; i < Array.getLength(value); i++) {
Object v = Array.get(value, i);
if (v == null) {
continue;
}
if (v instanceof File) {
File f = (File) v;
meBuilder.addBinaryBody(key, f,
ContentType.APPLICATION_OCTET_STREAM,
f.getName());
}
else if (v instanceof ContentBody) {
meBuilder.addPart(key, (ContentBody) v);
}
else {
meBuilder.addTextBody(key, v.toString(),
ContentType.create("text/plain",
Charset.forName(postCharset)));
}
}
}
else {
meBuilder.addTextBody(
key,
value.toString(),
ContentType.create("text/plain",
Charset.forName(postCharset)));
}
}
reqBuilder.setEntity(meBuilder.build());
}
else if (bodyType == 2) {
for (Entry<String, Object> entry : parameters.entrySet()) {
Object value = entry.getValue();
if (value == null) {
continue;
}
if (value instanceof HttpEntity) {
reqBuilder.setEntity((HttpEntity) value);
}
}
}
else if (bodyType == 3) {
StringEntity se = new StringEntity((String) parameters.get("json"));
se.setContentType("application/json");
reqBuilder.setEntity(se);
}
else {
List<NameValuePair> nvps = new ArrayList<NameValuePair>(
parameters.size());
for (Entry<String, Object> entry : parameters.entrySet()) {
String key = entry.getKey();
Object value = entry.getValue();
if (key == null || value == null) {
continue;
}
if (value.getClass().isArray()) {
for (int i = 0; i < Array.getLength(value); i++) {
Object v = Array.get(value, i);
if (v == null) {
continue;
}
nvps.add(new BasicNameValuePair(key, v.toString()));
}
}
else {
nvps.add(new BasicNameValuePair(key, value.toString()));
}
}
if (StringUtils.isBlank(postCharset)) {
postCharset = "utf-8";
}
try {
reqBuilder.setEntity(new UrlEncodedFormEntity(nvps,
postCharset));
System.out.println(IOUtils.toString(reqBuilder.getEntity().getContent()));
}
catch (Exception ex) {
}
}
}
HttpUriRequest post = reqBuilder.build();
String host = post.getURI().getHost();
if (usingReferer) {
if (StringUtils.isBlank(referer)) {
post.setHeader("Referer", "http://" + host);
}
else {
post.setHeader("Referer", referer);
}
}
return post;
}