1 |
package dk.thoerup.spejdernetscraper; |
package dk.thoerup.spejdernetscraper; |
2 |
|
|
3 |
|
import java.util.List; |
4 |
import java.util.Map; |
import java.util.Map; |
5 |
import java.util.TreeMap; |
import java.util.TreeMap; |
6 |
|
|
|
import javax.servlet.http.HttpUtils; |
|
|
|
|
|
import org.jsoup.Connection.Method; |
|
7 |
import org.jsoup.Connection.Response; |
import org.jsoup.Connection.Response; |
8 |
import org.jsoup.Jsoup; |
import org.jsoup.Jsoup; |
9 |
|
import org.jsoup.nodes.Document; |
10 |
|
import org.jsoup.nodes.FormElement; |
11 |
|
|
12 |
import dk.thoerup.genericjavautils.HttpUtil; |
import dk.thoerup.genericjavautils.HttpUtil; |
13 |
|
|
15 |
|
|
16 |
@Override |
@Override |
17 |
public void run() { |
public void run() { |
18 |
long start = System.currentTimeMillis(); |
|
19 |
ResultBuffer rb = ResultBuffer.getResultBuffer(); |
ResultBuffer rb = ResultBuffer.getResultBuffer(); |
20 |
try { |
try { |
21 |
|
|
22 |
rb.clear(); |
rb.clear(); |
23 |
rb.addString(start, "Starting"); |
rb.addString("Starting"); |
24 |
|
|
25 |
|
|
26 |
Response res = Jsoup.connect("http://medlemssystem.spejdernet.dk/login.aspx?ReturnUrl=%2f").execute(); |
Response res = Jsoup.connect("http://medlemssystem.spejdernet.dk/login.aspx?ReturnUrl=%2f").execute(); |
27 |
Map<String, String> cookies = res.cookies(); |
Map<String, String> cookies = res.cookies(); |
28 |
|
|
29 |
rb.addString(start, "Got login page"); |
Document loginPage = res.parse(); |
30 |
|
|
31 |
|
rb.addString("Got login page"); |
32 |
|
|
33 |
|
|
34 |
|
|
35 |
|
FormElement loginForm = (FormElement) loginPage.getElementById("aspnetForm"); |
36 |
|
loginForm.getElementById("ctl00_main_loginForm__txtUserName").attr("value","torbenhoerupnielsen"); |
37 |
|
loginForm.getElementById("ctl00_main_loginForm__txtPassword").attr("value","Pwspejder2013"); |
38 |
|
System.out.println( loginForm.formData() ); |
39 |
|
|
40 |
res = Jsoup |
res = loginForm.submit().execute(); |
41 |
.connect("http://medlemssystem.spejdernet.dk/Login.aspx?changeuser=1") |
//System.out.println( res.body() ); //DEBUG |
|
.data("ctl00$main$loginForm$_txtUserName", "torbenhoerupnielsen", |
|
|
"ctl00$main$loginForm$_txtPassword", "Pwspejder2013", |
|
|
"ctl00$main$loginForm$_buttonLogin", "Login", |
|
|
"__EVENTTARGET", "", |
|
|
"__EVENTARGUMENT", "", |
|
|
"__VIEWSTATE", "/wEPDwUJNjg2MjA4NzU2D2QWAmYPZBYCAgMPZBYGAgEPDxYCHgdWaXNpYmxlaGRkAgIPZBYIAgkPDxYCHwBoZGQCDQ8PFgIfAGhkZAIPDw8WAh8AaGRkAhMPDxYCHwBoZGQCAw9kFgICAQ9kFgRmDw8WAh8AaGRkAgYPDxYCHwBoZGRk8Vg6q/CGyZv+RzwBkdn5NiLJnoGAlWgM6iWe62LJuHM=", |
|
|
"ctl00$ctl04$hiddenTab", "" |
|
|
) |
|
|
.method(Method.POST) |
|
|
.cookies(cookies) |
|
|
.execute(); |
|
42 |
|
|
43 |
rb.addString(start, "Login OK"); |
rb.addString("Login OK"); |
44 |
//This will get you cookies |
//This will get you cookies |
45 |
cookies.putAll( res.cookies() ); |
cookies.putAll( res.cookies() ); |
46 |
//printMap(cookies); |
//printMap(cookies); |
|
|
|
47 |
|
|
48 |
String dataCsvUrl = "http://medlemssystem.spejdernet.dk/csvx.ashx?q=JQBOZXRtZXN0ZXIuS0ZVTS5NZW1iZXIsIE5ldG1lc3Rlci5LRlVNPgB4LlVuaXQuR3JvdXAuSWQgPT0gImd1aWQ6NWMwY2Y2MzgtZGM4Ny00MTNmLTkxYjYtOWM3MzAwYzcwMTM3Ig==&type=Member&view=csv"; |
String dataCsvUrl = "http://medlemssystem.spejdernet.dk/csvx.ashx?q=JQBOZXRtZXN0ZXIuS0ZVTS5NZW1iZXIsIE5ldG1lc3Rlci5LRlVNPgB4LlVuaXQuR3JvdXAuSWQgPT0gImd1aWQ6NWMwY2Y2MzgtZGM4Ny00MTNmLTkxYjYtOWM3MzAwYzcwMTM3Ig==&type=Member&view=csv"; |
49 |
//And this is the easieste way I've found to remain in session |
//And this is the easieste way I've found to remain in session |
50 |
Response data = Jsoup.connect(dataCsvUrl).cookies(cookies).timeout(10000).execute(); |
Response data = Jsoup.connect(dataCsvUrl).cookies(cookies).timeout(10000).execute(); |
51 |
if ( ! data.contentType().equalsIgnoreCase("text/x-csv; charset=iso-8859-1") ) { |
if ( ! data.contentType().equalsIgnoreCase("text/x-csv; charset=iso-8859-1") ) { |
52 |
rb.addString(start, "Data.csv - has the wrong content type: " + data.contentType()); |
|
53 |
|
rb.addString("Data.csv - has the wrong content type: " + data.contentType()); |
54 |
return; |
return; |
55 |
} |
} |
56 |
String dataCsv = data.body(); |
String dataCsv = data.body(); |
57 |
rb.addString(start, "Got data.csv"); |
rb.addString("Got data.csv"); |
58 |
|
|
59 |
String rollerCsvUrl = "http://medlemssystem.spejdernet.dk/DataExport.aspx?Id=5c0cf638-dc87-413f-91b6-9c7300c70137&listid=e8c5ae9d-5ea7-4a00-bea4-a0ce00ea891e&execute=true"; |
String rollerCsvUrl = "http://medlemssystem.spejdernet.dk/DataExport.aspx?Id=5c0cf638-dc87-413f-91b6-9c7300c70137&listid=e8c5ae9d-5ea7-4a00-bea4-a0ce00ea891e&execute=true"; |
60 |
Response roller = Jsoup.connect(rollerCsvUrl).cookies(cookies).timeout(10000).execute(); |
Response roller = Jsoup.connect(rollerCsvUrl).cookies(cookies).timeout(10000).execute(); |
61 |
if ( ! roller.contentType().equalsIgnoreCase("text/x-csv; charset=iso-8859-1") ) { |
if ( ! roller.contentType().equalsIgnoreCase("text/x-csv; charset=iso-8859-1") ) { |
62 |
rb.addString(start, "Roller.csv - has the wrong content type: " + roller.contentType()); |
rb.addString("Roller.csv - has the wrong content type: " + roller.contentType()); |
63 |
return; |
return; |
64 |
} |
} |
65 |
String rollerCsv = roller.body(); |
String rollerCsv = roller.body(); |
66 |
rb.addString(start, "Got roller.csv"); |
rb.addString("Got roller.csv"); |
67 |
|
|
68 |
Map<String,String> postData = new TreeMap<String,String>(); |
Map<String,String> postData = new TreeMap<String,String>(); |
69 |
postData.put("roller", rollerCsv); |
postData.put("roller", rollerCsv); |
73 |
byte resp[] = HttpUtil.postContent("http://horsensspejder.t-hoerup.dk/data/postdata.php", params, 3000); |
byte resp[] = HttpUtil.postContent("http://horsensspejder.t-hoerup.dk/data/postdata.php", params, 3000); |
74 |
|
|
75 |
String response = new String(resp); |
String response = new String(resp); |
76 |
rb.addString(start, "POST Response: " + response); |
rb.addString("POST Response: " + response); |
77 |
|
|
78 |
rb.addString(start, "Done !"); |
rb.addString("Done !"); |
79 |
|
|
80 |
} catch (Exception e) { |
} catch (Exception e) { |
81 |
rb.addString(start, "Error occurred: " + e.toString() ); |
rb.addString("Error occurred: " + e.toString() ); |
82 |
} |
} |
83 |
} |
} |
84 |
|
|
95 |
System.out.println( ResultBuffer.getResultBuffer().getString() ); |
System.out.println( ResultBuffer.getResultBuffer().getString() ); |
96 |
} |
} |
97 |
|
|
98 |
|
|
99 |
|
/* |
100 |
|
* |
101 |
|
* Just fore reference, here's the old login method |
102 |
|
res = Jsoup |
103 |
|
.connect("http://medlemssystem.spejdernet.dk/Login.aspx?changeuser=1") |
104 |
|
.data("ctl00$main$loginForm$_txtUserName", "<enter-username-here>", |
105 |
|
"ctl00$main$loginForm$_txtPassword", "<enter-password-here>", |
106 |
|
"ctl00$main$loginForm$_buttonLogin", "Login", |
107 |
|
"__EVENTTARGET", "", |
108 |
|
"__EVENTARGUMENT", "", |
109 |
|
"__VIEWSTATE", viewState.attr("value"), |
110 |
|
"ctl00$ctl04$hiddenTab", "" |
111 |
|
) |
112 |
|
.method(Method.POST) |
113 |
|
.cookies(cookies) |
114 |
|
.execute();*/ |
115 |
|
|
116 |
} |
} |