thoerup/spejdernetscraper/ScraperWorker.java

package dk.thoerup.spejdernetscraper;

import java.util.Map;
import java.util.TreeMap;

import org.apache.commons.lang3.exception.ExceptionUtils;
import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.FormElement;

import dk.thoerup.genericjavautils.HttpUtil;

public class ScraperWorker implements Runnable {

        @Override
        public void run() {
                
                ResultBuffer rb = ResultBuffer.getResultBuffer();
                try {
                                                
                        rb.clear();                     
                        rb.addString("Starting");
                        
                        
                        Response  res = Jsoup.connect("http://medlemssystem.spejdernet.dk/login.aspx?ReturnUrl=%2f").execute();
                        Map<String, String> cookies = res.cookies();
                        
                        Document loginPage = res.parse();                                       
                        
                        rb.addString("Got login page");
                                                

                        
                        FormElement loginForm = (FormElement) loginPage.getElementById("aspnetForm");                   
                        loginForm.getElementById("ctl00_main_loginForm__txtUserName").attr("value","torbenhoerupnielsen");
                        loginForm.getElementById("ctl00_main_loginForm__txtPassword").attr("value","Pwspejder2013");
                        System.out.println( loginForm.formData() );
                        
                        res = loginForm.submit().execute();
                        //System.out.println( res.body() ); //DEBUG
        
                        rb.addString("Login OK");
                        //This will get you cookies
                        cookies.putAll( res.cookies() );
                        //printMap(cookies);    
                        
                        // /////////////////////////////////
                        // Data.csv
        
                        String dataCsvUrl = "http://medlemssystem.spejdernet.dk/csvx.ashx?q=JQBOZXRtZXN0ZXIuS0ZVTS5NZW1iZXIsIE5ldG1lc3Rlci5LRlVNPgB4LlVuaXQuR3JvdXAuSWQgPT0gImd1aWQ6NWMwY2Y2MzgtZGM4Ny00MTNmLTkxYjYtOWM3MzAwYzcwMTM3Ig==&type=Member&view=csv";
                        //And this is the easieste way I've found to remain in session
                        Response data = Jsoup.connect(dataCsvUrl).cookies(cookies).timeout(10000).execute();
                        if ( ! data.contentType().equalsIgnoreCase("text/x-csv; charset=iso-8859-1") ) {
                                
                                rb.addString("Data.csv - has the wrong content type: " + data.contentType());
                                return;
                        }                       
                        String dataCsv = data.body();
                        rb.addString("Got data.csv");
                        
                        // /////////////////////////////////
                        // Roller.csv
                        /*
                        String rollerCsvUrl = "http://medlemssystem.spejdernet.dk/DataExport.aspx?Id=5c0cf638-dc87-413f-91b6-9c7300c70137&listid=e8c5ae9d-5ea7-4a00-bea4-a0ce00ea891e&execute=true";
                        Response roller = Jsoup.connect(rollerCsvUrl).cookies(cookies).timeout(10000).execute();
                        if ( ! roller.contentType().equalsIgnoreCase("text/x-csv; charset=iso-8859-1") ) {
                                rb.addString("Roller.csv - has the wrong content type: " + roller.contentType());
                                return;
                        }
                        String rollerCsv = roller.body();
                        rb.addString("Got roller.csv");*/
                        String rollerCsv = ""; //Temporary fix for PrematureEof exception thingie
                        
                        Map<String,String> postData = new TreeMap<String,String>();
                        postData.put("roller", rollerCsv);
                        postData.put("data", dataCsv);
                        
                        String params = HttpUtil.encodeParams( postData );
                        byte resp[] = HttpUtil.postContent("http://horsensspejder.t-hoerup.dk/data/postdata.php", params, 3000);
                        
                        String response = new String(resp);
                        rb.addString("POST Response: " + response);
                        
                        rb.addString("Done !");
                        
                } catch (Exception e) {
                        rb.addString("Error occurred: " + e.toString() );
                        rb.addString( ExceptionUtils.getStackTrace(e) );
                }
        }       
        
        static void printMap(Map<String,String> map) {
                for(String key : map.keySet()) {
                        System.out.println( key + ": " +map.get(key) );
                }
                
        }

        public static void main(String[] args) throws Exception {
                new ScraperWorker().run();
                
                System.out.println( ResultBuffer.getResultBuffer().getString() );
        }

        
        /*
         * 
         * Just fore reference, here's the old login method
        res = Jsoup
                        .connect("http://medlemssystem.spejdernet.dk/Login.aspx?changeuser=1")
                        .data("ctl00$main$loginForm$_txtUserName", "<enter-username-here>", 
                                        "ctl00$main$loginForm$_txtPassword", "<enter-password-here>",
                                        "ctl00$main$loginForm$_buttonLogin", "Login",
                                        "__EVENTTARGET", "",
                                        "__EVENTARGUMENT", "",
                                        "__VIEWSTATE", viewState.attr("value"),
                                        "ctl00$ctl04$hiddenTab", ""
                                        )
                                        .method(Method.POST)
                                        .cookies(cookies)
                                        .execute();*/

}
1	torben	2041	package dk.thoerup.spejdernetscraper;
2
3			import java.util.Map;
4			import java.util.TreeMap;
5
6	torben	2251	import org.apache.commons.lang3.exception.ExceptionUtils;
7	torben	2041	import org.jsoup.Connection.Response;
8			import org.jsoup.Jsoup;
9	torben	2104	import org.jsoup.nodes.Document;
10	torben	2138	import org.jsoup.nodes.FormElement;
11	torben	2041
12			import dk.thoerup.genericjavautils.HttpUtil;
13
14			public class ScraperWorker implements Runnable {
15
16			@Override
17			public void run() {
18	torben	2072
19	torben	2041	ResultBuffer rb = ResultBuffer.getResultBuffer();
20			try {
21
22			rb.clear();
23	torben	2072	rb.addString("Starting");
24	torben	2041
25
26			Response res = Jsoup.connect("http://medlemssystem.spejdernet.dk/login.aspx?ReturnUrl=%2f").execute();
27			Map<String, String> cookies = res.cookies();
28
29	torben	2138	Document loginPage = res.parse();
30	torben	2104
31	torben	2138	rb.addString("Got login page");
32	torben	2139
33
34	torben	2104
35	torben	2139	FormElement loginForm = (FormElement) loginPage.getElementById("aspnetForm");
36	torben	2138	loginForm.getElementById("ctl00_main_loginForm__txtUserName").attr("value","torbenhoerupnielsen");
37			loginForm.getElementById("ctl00_main_loginForm__txtPassword").attr("value","Pwspejder2013");
38			System.out.println( loginForm.formData() );
39	torben	2041
40	torben	2138	res = loginForm.submit().execute();
41			//System.out.println( res.body() ); //DEBUG
42	torben	2041
43	torben	2072	rb.addString("Login OK");
44	torben	2041	//This will get you cookies
45			cookies.putAll( res.cookies() );
46	torben	2138	//printMap(cookies);
47	torben	2252
48			// /////////////////////////////////
49			// Data.csv
50	torben	2041
51			String dataCsvUrl = "http://medlemssystem.spejdernet.dk/csvx.ashx?q=JQBOZXRtZXN0ZXIuS0ZVTS5NZW1iZXIsIE5ldG1lc3Rlci5LRlVNPgB4LlVuaXQuR3JvdXAuSWQgPT0gImd1aWQ6NWMwY2Y2MzgtZGM4Ny00MTNmLTkxYjYtOWM3MzAwYzcwMTM3Ig==&type=Member&view=csv";
52			//And this is the easieste way I've found to remain in session
53	torben	2046	Response data = Jsoup.connect(dataCsvUrl).cookies(cookies).timeout(10000).execute();
54			if ( ! data.contentType().equalsIgnoreCase("text/x-csv; charset=iso-8859-1") ) {
55	torben	2102
56	torben	2072	rb.addString("Data.csv - has the wrong content type: " + data.contentType());
57	torben	2046	return;
58			}
59	torben	2041	String dataCsv = data.body();
60	torben	2072	rb.addString("Got data.csv");
61	torben	2041
62	torben	2252	// /////////////////////////////////
63			// Roller.csv
64			/*
65	torben	2041	String rollerCsvUrl = "http://medlemssystem.spejdernet.dk/DataExport.aspx?Id=5c0cf638-dc87-413f-91b6-9c7300c70137&listid=e8c5ae9d-5ea7-4a00-bea4-a0ce00ea891e&execute=true";
66			Response roller = Jsoup.connect(rollerCsvUrl).cookies(cookies).timeout(10000).execute();
67	torben	2046	if ( ! roller.contentType().equalsIgnoreCase("text/x-csv; charset=iso-8859-1") ) {
68	torben	2072	rb.addString("Roller.csv - has the wrong content type: " + roller.contentType());
69	torben	2046	return;
70			}
71	torben	2041	String rollerCsv = roller.body();
72	torben	2252	rb.addString("Got roller.csv");*/
73			String rollerCsv = ""; //Temporary fix for PrematureEof exception thingie
74	torben	2041
75			Map<String,String> postData = new TreeMap<String,String>();
76			postData.put("roller", rollerCsv);
77			postData.put("data", dataCsv);
78
79			String params = HttpUtil.encodeParams( postData );
80			byte resp[] = HttpUtil.postContent("http://horsensspejder.t-hoerup.dk/data/postdata.php", params, 3000);
81
82			String response = new String(resp);
83	torben	2072	rb.addString("POST Response: " + response);
84	torben	2041
85	torben	2072	rb.addString("Done !");
86	torben	2042
87	torben	2041	} catch (Exception e) {
88	torben	2072	rb.addString("Error occurred: " + e.toString() );
89	torben	2251	rb.addString( ExceptionUtils.getStackTrace(e) );
90	torben	2041	}
91			}
92
93			static void printMap(Map<String,String> map) {
94			for(String key : map.keySet()) {
95			System.out.println( key + ": " +map.get(key) );
96			}
97
98			}
99
100			public static void main(String[] args) throws Exception {
101			new ScraperWorker().run();
102
103			System.out.println( ResultBuffer.getResultBuffer().getString() );
104			}
105
106	torben	2138
107			/*
108			*
109			* Just fore reference, here's the old login method
110			res = Jsoup
111			.connect("http://medlemssystem.spejdernet.dk/Login.aspx?changeuser=1")
112			.data("ctl00$main$loginForm$_txtUserName", "<enter-username-here>",
113			"ctl00$main$loginForm$_txtPassword", "<enter-password-here>",
114			"ctl00$main$loginForm$_buttonLogin", "Login",
115			"__EVENTTARGET", "",
116			"__EVENTARGUMENT", "",
117			"__VIEWSTATE", viewState.attr("value"),
118			"ctl00$ctl04$hiddenTab", ""
119			)
120			.method(Method.POST)
121			.cookies(cookies)
122			.execute();*/
123	torben	2041
124			}