Proxy를 이용한 HTML 소스 가져오기

Proxy를 이용해서 HTML 소스를 가져와야 하는 경우가 있을 때 유용할 것 같습니다. 

개발 세상만사님의 스프링노트에서 가져왔습니다. 

나중을 위해 소스를 옮겨 봅니다. 중간에 "아이피", "포트"는 public proxy를 검색하셔서 지정해 주면 될 것 같네요. 

단, Proxy는 확실히 속도가 느린 것 같습니다. 

또한 Proxy 서버를 통할 경우, HTTP 프로토콜로 주고받은 데이터가 모두 Proxy 서버에 저장되므로 주의하셔야 합니다. 

view plaincopy to clipboardprint?

1. package gathering;  
2.   
3. import java.io.BufferedReader;  
4. import java.io.InputStreamReader;  
5. import java.net.HttpURLConnection;  
6. import java.net.InetSocketAddress;  
7. import java.net.Proxy;  
8. import java.net.URL;  
9. import java.net.URLEncoder;  
10.   
11. public class Proxy1 {  
12.   
13.     public static void main(String[] args) throws Exception{  
14.   
15.         String strKeyword    = URLEncoder.encode("신발", "UTF-8");  
16.         String strUrl            = "http://search.naver.com/search.naver?where=nexearch&sm=osd&ie=UTF-8&query="+strKeyword;  
17.         Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress("아이피", 포트));  
18.         URL url = new URL(strUrl);  
19.         HttpURLConnection uc = (HttpURLConnection)url.openConnection(proxy);  
20.         uc.setRequestProperty("User-Agent","Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2)");  
21.         uc.setRequestProperty("Accept-Language", "ko-KR");  
22.         uc.setRequestProperty("Method", "GET");  
23.         uc.setRequestProperty("Accept", "image/gif, image/xxbitmap, image/jpeg, image/pjpeg,application/xshockwaveflash, application/vnd.msexcel,application/vnd.mspowerpoint, application/msword, */*");  
24.         uc.setRequestProperty("Accept-Charset","UTF-8");  
25.         uc.setRequestProperty("Referer","http://www.kr.yahoo.com/");         
26.         uc.connect();  
27.   
28.         String line;  
29.         StringBuffer tmp         = new StringBuffer();  
30.         BufferedReader br = new BufferedReader(new InputStreamReader(uc.getInputStream(), "EUC-KR"));  
31.         while ((line = br.readLine()) != null){  
32.             tmp.append(line);  
33.         }  
34.          
35.         System.out.println(tmp.toString());  
36.     }  
37. }  

package gathering; import java.io.BufferedReader; import java.io.InputStreamReader; import java.net.HttpURLConnection; import java.net.InetSocketAddress; import java.net.Proxy; import java.net.URL; import java.net.URLEncoder; public class Proxy1 { public static void main(String[] args) throws Exception{ String strKeyword = URLEncoder.encode("신발", "UTF-8"); String strUrl = "http://search.naver.com/search.naver?where=nexearch&sm=osd&ie=UTF-8&query="+strKeyword; Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress("아이피", 포트)); URL url = new URL(strUrl); HttpURLConnection uc = (HttpURLConnection)url.openConnection(proxy); uc.setRequestProperty("User-Agent","Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2)"); uc.setRequestProperty("Accept-Language", "ko-KR"); uc.setRequestProperty("Method", "GET"); uc.setRequestProperty("Accept", "image/gif, image/xxbitmap, image/jpeg, image/pjpeg,application/xshockwaveflash, application/vnd.msexcel,application/vnd.mspowerpoint, application/msword, */*"); uc.setRequestProperty("Accept-Charset","UTF-8"); uc.setRequestProperty("Referer","http://www.kr.yahoo.com/"); uc.connect(); String line; StringBuffer tmp = new StringBuffer(); BufferedReader br = new BufferedReader(new InputStreamReader(uc.getInputStream(), "EUC-KR")); while ((line = br.readLine()) != null){ tmp.append(line); } System.out.println(tmp.toString()); } }