thoerup/spejdernetscraper/ScraperWorker.java

package dk.thoerup.spejdernetscraper;

import java.util.Map;
import java.util.TreeMap;

import javax.servlet.http.HttpUtils;

import org.jsoup.Connection.Method;
import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;

import dk.thoerup.genericjavautils.HttpUtil;

public class ScraperWorker implements Runnable {

        @Override
        public void run() {
                long start = System.currentTimeMillis();
                ResultBuffer rb = ResultBuffer.getResultBuffer();
                try {
                                                
                        rb.clear();                     
                        rb.addString(start, "Starting");
                        
                        
                        Response  res = Jsoup.connect("http://medlemssystem.spejdernet.dk/login.aspx?ReturnUrl=%2f").execute();
                        Map<String, String> cookies = res.cookies();
                        
                        rb.addString(start, "Got login page");
                        
                        res = Jsoup
                                        .connect("http://medlemssystem.spejdernet.dk/Login.aspx?changeuser=1")
                                        .data("ctl00$main$loginForm$_txtUserName", "torbenhoerupnielsen", 
                                                        "ctl00$main$loginForm$_txtPassword", "Pwspejder2013",
                                                        "ctl00$main$loginForm$_buttonLogin", "Login",
                                                        "__EVENTTARGET", "",
                                                        "__EVENTARGUMENT", "",
                                                        "__VIEWSTATE", "/wEPDwUJNjg2MjA4NzU2D2QWAmYPZBYCAgMPZBYGAgEPDxYCHgdWaXNpYmxlaGRkAgIPZBYIAgkPDxYCHwBoZGQCDQ8PFgIfAGhkZAIPDw8WAh8AaGRkAhMPDxYCHwBoZGQCAw9kFgICAQ9kFgRmDw8WAh8AaGRkAgYPDxYCHwBoZGRk8Vg6q/CGyZv+RzwBkdn5NiLJnoGAlWgM6iWe62LJuHM=",
                                                        "ctl00$ctl04$hiddenTab", ""
                                                        )
                                                        .method(Method.POST)
                                                        .cookies(cookies)
                                                        .execute();
        
                        rb.addString(start, "Login OK");
                        //This will get you cookies
                        cookies.putAll( res.cookies() );
                        //printMap(cookies);
                        
        
                        String dataCsvUrl = "http://medlemssystem.spejdernet.dk/csvx.ashx?q=JQBOZXRtZXN0ZXIuS0ZVTS5NZW1iZXIsIE5ldG1lc3Rlci5LRlVNPgB4LlVuaXQuR3JvdXAuSWQgPT0gImd1aWQ6NWMwY2Y2MzgtZGM4Ny00MTNmLTkxYjYtOWM3MzAwYzcwMTM3Ig==&type=Member&view=csv";
                        //And this is the easieste way I've found to remain in session
                        Response data = Jsoup.connect(dataCsvUrl).cookies(cookies).timeout(10000).execute();
                        if ( ! data.contentType().equalsIgnoreCase("text/x-csv; charset=iso-8859-1") ) {
                                rb.addString(start, "Data.csv - has the wrong content type: " + data.contentType());
                                return;
                        }                       
                        String dataCsv = data.body();
                        rb.addString(start, "Got data.csv");
                        
                        String rollerCsvUrl = "http://medlemssystem.spejdernet.dk/DataExport.aspx?Id=5c0cf638-dc87-413f-91b6-9c7300c70137&listid=e8c5ae9d-5ea7-4a00-bea4-a0ce00ea891e&execute=true";
                        Response roller = Jsoup.connect(rollerCsvUrl).cookies(cookies).timeout(10000).execute();
                        if ( ! roller.contentType().equalsIgnoreCase("text/x-csv; charset=iso-8859-1") ) {
                                rb.addString(start, "Roller.csv - has the wrong content type: " + roller.contentType());
                                return;
                        }
                        String rollerCsv = roller.body();
                        rb.addString(start, "Got roller.csv");
                        
                        Map<String,String> postData = new TreeMap<String,String>();
                        postData.put("roller", rollerCsv);
                        postData.put("data", dataCsv);
                        
                        String params = HttpUtil.encodeParams( postData );
                        byte resp[] = HttpUtil.postContent("http://horsensspejder.t-hoerup.dk/data/postdata.php", params, 3000);
                        
                        String response = new String(resp);
                        rb.addString(start, "POST Response: " + response);
                        
                        rb.addString(start, "Done !");
                        
                } catch (Exception e) {
                        rb.addString(start, "Error occurred: " + e.toString() );
                }
        }       
        
        static void printMap(Map<String,String> map) {
                for(String key : map.keySet()) {
                        System.out.println( key + ": " +map.get(key) );
                }
                
        }

        public static void main(String[] args) throws Exception {
                new ScraperWorker().run();
                
                System.out.println( ResultBuffer.getResultBuffer().getString() );
        }


}
1	torben	2041	package dk.thoerup.spejdernetscraper;
2
3			import java.util.Map;
4			import java.util.TreeMap;
5
6			import javax.servlet.http.HttpUtils;
7
8			import org.jsoup.Connection.Method;
9			import org.jsoup.Connection.Response;
10			import org.jsoup.Jsoup;
11
12			import dk.thoerup.genericjavautils.HttpUtil;
13
14			public class ScraperWorker implements Runnable {
15
16			@Override
17			public void run() {
18			long start = System.currentTimeMillis();
19			ResultBuffer rb = ResultBuffer.getResultBuffer();
20			try {
21
22			rb.clear();
23			rb.addString(start, "Starting");
24
25
26			Response res = Jsoup.connect("http://medlemssystem.spejdernet.dk/login.aspx?ReturnUrl=%2f").execute();
27			Map<String, String> cookies = res.cookies();
28
29			rb.addString(start, "Got login page");
30
31			res = Jsoup
32			.connect("http://medlemssystem.spejdernet.dk/Login.aspx?changeuser=1")
33			.data("ctl00$main$loginForm$_txtUserName", "torbenhoerupnielsen",
34			"ctl00$main$loginForm$_txtPassword", "Pwspejder2013",
35			"ctl00$main$loginForm$_buttonLogin", "Login",
36			"__EVENTTARGET", "",
37			"__EVENTARGUMENT", "",
38			"__VIEWSTATE", "/wEPDwUJNjg2MjA4NzU2D2QWAmYPZBYCAgMPZBYGAgEPDxYCHgdWaXNpYmxlaGRkAgIPZBYIAgkPDxYCHwBoZGQCDQ8PFgIfAGhkZAIPDw8WAh8AaGRkAhMPDxYCHwBoZGQCAw9kFgICAQ9kFgRmDw8WAh8AaGRkAgYPDxYCHwBoZGRk8Vg6q/CGyZv+RzwBkdn5NiLJnoGAlWgM6iWe62LJuHM=",
39			"ctl00$ctl04$hiddenTab", ""
40			)
41			.method(Method.POST)
42			.cookies(cookies)
43			.execute();
44
45			rb.addString(start, "Login OK");
46			//This will get you cookies
47			cookies.putAll( res.cookies() );
48			//printMap(cookies);
49
50
51			String dataCsvUrl = "http://medlemssystem.spejdernet.dk/csvx.ashx?q=JQBOZXRtZXN0ZXIuS0ZVTS5NZW1iZXIsIE5ldG1lc3Rlci5LRlVNPgB4LlVuaXQuR3JvdXAuSWQgPT0gImd1aWQ6NWMwY2Y2MzgtZGM4Ny00MTNmLTkxYjYtOWM3MzAwYzcwMTM3Ig==&type=Member&view=csv";
52			//And this is the easieste way I've found to remain in session
53	torben	2046	Response data = Jsoup.connect(dataCsvUrl).cookies(cookies).timeout(10000).execute();
54			if ( ! data.contentType().equalsIgnoreCase("text/x-csv; charset=iso-8859-1") ) {
55			rb.addString(start, "Data.csv - has the wrong content type: " + data.contentType());
56			return;
57			}
58	torben	2041	String dataCsv = data.body();
59			rb.addString(start, "Got data.csv");
60
61			String rollerCsvUrl = "http://medlemssystem.spejdernet.dk/DataExport.aspx?Id=5c0cf638-dc87-413f-91b6-9c7300c70137&listid=e8c5ae9d-5ea7-4a00-bea4-a0ce00ea891e&execute=true";
62			Response roller = Jsoup.connect(rollerCsvUrl).cookies(cookies).timeout(10000).execute();
63	torben	2046	if ( ! roller.contentType().equalsIgnoreCase("text/x-csv; charset=iso-8859-1") ) {
64			rb.addString(start, "Roller.csv - has the wrong content type: " + roller.contentType());
65			return;
66			}
67	torben	2041	String rollerCsv = roller.body();
68			rb.addString(start, "Got roller.csv");
69
70			Map<String,String> postData = new TreeMap<String,String>();
71			postData.put("roller", rollerCsv);
72			postData.put("data", dataCsv);
73
74			String params = HttpUtil.encodeParams( postData );
75			byte resp[] = HttpUtil.postContent("http://horsensspejder.t-hoerup.dk/data/postdata.php", params, 3000);
76
77			String response = new String(resp);
78			rb.addString(start, "POST Response: " + response);
79
80	torben	2042	rb.addString(start, "Done !");
81
82	torben	2041	} catch (Exception e) {
83			rb.addString(start, "Error occurred: " + e.toString() );
84			}
85			}
86
87			static void printMap(Map<String,String> map) {
88			for(String key : map.keySet()) {
89			System.out.println( key + ": " +map.get(key) );
90			}
91
92			}
93
94			public static void main(String[] args) throws Exception {
95			new ScraperWorker().run();
96
97			System.out.println( ResultBuffer.getResultBuffer().getString() );
98			}
99
100
101			}