thoerup/spejdernetscraper/OKMobilSaldo.java

package dk.thoerup.spejdernetscraper;

import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.net.HttpURLConnection;
import java.net.URLEncoder;
import java.util.Collection;
import java.util.List;
import java.util.Map;

import org.jsoup.Connection;
import org.jsoup.Connection.Method;
import org.jsoup.Connection.Request;
import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

public class OKMobilSaldo {

        final String mobil = "71750247";
        final String password = "gc93ehcv";

        final String authKey = ".ASPXAUTH"; 

        String chromeUserAgent = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36";


        public String hentSaldo() {
                try {
                        String frontpageUrl = "https://www.ok-mobil.dk/";
                        String loginUrl = "https://www.ok-mobil.dk/log-paa";
                        String accountUrl = "https://www.ok-mobil.dk/min-konto";

                        Response  res = Jsoup.connect(frontpageUrl)
                                        .userAgent(chromeUserAgent)
                                        .execute();

                        Map<String, String> cookies = res.cookies();


                        Request req = Jsoup
                                        .connect( loginUrl )
                                        .userAgent(chromeUserAgent)
                                        .referrer(frontpageUrl)
                                        .data("phoneNumber", mobil, 
                                                        "password", password
                                                        )
                                                        .method(Method.POST)
                                                        .cookies(cookies)
                                                        .request();


                        HttpURLConnection conn = createConnection(req);
                        conn.connect();
                        if (req.method() == Connection.Method.POST)
                                writePost(req.data(), conn.getOutputStream());
                        
                        int status = conn.getResponseCode();
                        
                        String auth = getAuthHeader(conn.getHeaderFields() );
                        
                        System.out.println("AUTH: " + auth);
                        
                        cookies.put(authKey, auth);
                        
        
                        Document accountPage = Jsoup.connect(accountUrl)
                                        .cookies(cookies)
                                        .userAgent(chromeUserAgent)
                                        .get();
                        

                        Elements infoElems = accountPage.getElementsByClass("cashBalance");

                        System.out.println(accountPage.text() );

                        return infoElems.text();

                } catch (Exception e) {
                        return "Fejl: " + e.getMessage();
                }
        }
        
        
        // My helper functions
        String mapToString(Map<String,String> map) {
                StringBuilder sb = new StringBuilder();
                for (String key : map.keySet()) {
                        sb.append(key).append(": ").append(map.get(key)).append("\n");
                }
                return sb.toString();
        }
        
        String maplistToString(Map<String,List<String>> map) {
                StringBuilder sb = new StringBuilder();
                for (String key : map.keySet()) {
                        List<String> vals = map.get(key);
                        for (String val : vals) {
                                sb.append(key).append(": ").append(val).append("\n");
                        }
                }
                return sb.toString();
        }
        
        String getAuthHeader(Map<String,List<String>> map) {
                
                for (String key : map.keySet()) {
                        System.out.println("Key:" + key);
                        
                        if (key == null) //Warning key may be null 
                                continue;
                        
                        if (key.equalsIgnoreCase("Set-Cookie")) {
                                List<String> vals = map.get(key);
                                for (String val : vals) {                               
                                        String parts[] = val.split(";");
                                        String baseCookie[] = parts[0].split("=");
                                        
                                        String cookieName = baseCookie[0];
                                        System.out.println("Cookie:" + cookieName);
                                        
                                        if (cookieName.equalsIgnoreCase(authKey)) {
                                                if (baseCookie.length==2 && baseCookie[1].length() > 10)
                                                        return baseCookie[1];
                                        }
                                }
                        }
                }
                return null;
        }
        
        
        //Snipped from org.jsoup.helper.HttpConnection.java

        private static HttpURLConnection createConnection(Connection.Request req) throws IOException {
                HttpURLConnection conn = (HttpURLConnection) req.url().openConnection();
                conn.setRequestMethod(req.method().name());
                conn.setInstanceFollowRedirects(false); // don't rely on native redirection support
                conn.setConnectTimeout(req.timeout());
                conn.setReadTimeout(req.timeout());
                if (req.method() == Method.POST)
                        conn.setDoOutput(true);
                if (req.cookies().size() > 0)
                        conn.addRequestProperty("Cookie", getRequestCookieString(req));
                for (Map.Entry<String, String> header : req.headers().entrySet()) {
                        conn.addRequestProperty(header.getKey(), header.getValue());
                }
                return conn;
        }
        private static String getRequestCookieString(Connection.Request req) {
                StringBuilder sb = new StringBuilder();
                boolean first = true;
                for (Map.Entry<String, String> cookie : req.cookies().entrySet()) {
                        if (!first)
                                sb.append("; ");
                        else
                                first = false;
                        sb.append(cookie.getKey()).append('=').append(cookie.getValue());
                        // todo: spec says only ascii, no escaping / encoding defined. validate on set? or escape somehow here?
                }
                return sb.toString();
        }

        private static void writePost(Collection<Connection.KeyVal> data, OutputStream outputStream) throws IOException {
                OutputStreamWriter w = new OutputStreamWriter(outputStream, "UTF-8");
                boolean first = true;
                for (Connection.KeyVal keyVal : data) {
                        if (!first) 
                                w.append('&');
                        else
                                first = false;

                        w.write(URLEncoder.encode(keyVal.key(), "UTF-8"));
                        w.write('=');
                        w.write(URLEncoder.encode(keyVal.value(), "UTF-8"));
                }
                w.close();
        }
}
1	package dk.thoerup.spejdernetscraper;
2
3	import java.io.IOException;
4	import java.io.OutputStream;
5	import java.io.OutputStreamWriter;
6	import java.net.HttpURLConnection;
7	import java.net.URLEncoder;
8	import java.util.Collection;
9	import java.util.List;
10	import java.util.Map;
11
12	import org.jsoup.Connection;
13	import org.jsoup.Connection.Method;
14	import org.jsoup.Connection.Request;
15	import org.jsoup.Connection.Response;
16	import org.jsoup.Jsoup;
17	import org.jsoup.nodes.Document;
18	import org.jsoup.select.Elements;
19
20	public class OKMobilSaldo {
21
22	final String mobil = "71750247";
23	final String password = "gc93ehcv";
24
25	final String authKey = ".ASPXAUTH";
26
27	String chromeUserAgent = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36";
28
29
30
31	public String hentSaldo() {
32	try {
33	String frontpageUrl = "https://www.ok-mobil.dk/";
34	String loginUrl = "https://www.ok-mobil.dk/log-paa";
35	String accountUrl = "https://www.ok-mobil.dk/min-konto";
36
37	Response res = Jsoup.connect(frontpageUrl)
38	.userAgent(chromeUserAgent)
39	.execute();
40
41	Map<String, String> cookies = res.cookies();
42
43
44
45	Request req = Jsoup
46	.connect( loginUrl )
47	.userAgent(chromeUserAgent)
48	.referrer(frontpageUrl)
49	.data("phoneNumber", mobil,
50	"password", password
51	)
52	.method(Method.POST)
53	.cookies(cookies)
54	.request();
55
56
57	HttpURLConnection conn = createConnection(req);
58	conn.connect();
59	if (req.method() == Connection.Method.POST)
60	writePost(req.data(), conn.getOutputStream());
61
62	int status = conn.getResponseCode();
63
64	String auth = getAuthHeader(conn.getHeaderFields() );
65
66	System.out.println("AUTH: " + auth);
67
68	cookies.put(authKey, auth);
69
70
71
72	Document accountPage = Jsoup.connect(accountUrl)
73	.cookies(cookies)
74	.userAgent(chromeUserAgent)
75	.get();
76
77
78	Elements infoElems = accountPage.getElementsByClass("cashBalance");
79
80	System.out.println(accountPage.text() );
81
82	return infoElems.text();
83
84	} catch (Exception e) {
85	return "Fejl: " + e.getMessage();
86	}
87	}
88
89
90	// My helper functions
91	String mapToString(Map<String,String> map) {
92	StringBuilder sb = new StringBuilder();
93	for (String key : map.keySet()) {
94	sb.append(key).append(": ").append(map.get(key)).append("\n");
95	}
96	return sb.toString();
97	}
98
99	String maplistToString(Map<String,List<String>> map) {
100	StringBuilder sb = new StringBuilder();
101	for (String key : map.keySet()) {
102	List<String> vals = map.get(key);
103	for (String val : vals) {
104	sb.append(key).append(": ").append(val).append("\n");
105	}
106	}
107	return sb.toString();
108	}
109
110	String getAuthHeader(Map<String,List<String>> map) {
111
112	for (String key : map.keySet()) {
113	System.out.println("Key:" + key);
114
115	if (key == null) //Warning key may be null
116	continue;
117
118	if (key.equalsIgnoreCase("Set-Cookie")) {
119	List<String> vals = map.get(key);
120	for (String val : vals) {
121	String parts[] = val.split(";");
122	String baseCookie[] = parts[0].split("=");
123
124	String cookieName = baseCookie[0];
125	System.out.println("Cookie:" + cookieName);
126
127	if (cookieName.equalsIgnoreCase(authKey)) {
128	if (baseCookie.length==2 && baseCookie[1].length() > 10)
129	return baseCookie[1];
130	}
131	}
132	}
133	}
134	return null;
135	}
136
137
138
139	//Snipped from org.jsoup.helper.HttpConnection.java
140
141	private static HttpURLConnection createConnection(Connection.Request req) throws IOException {
142	HttpURLConnection conn = (HttpURLConnection) req.url().openConnection();
143	conn.setRequestMethod(req.method().name());
144	conn.setInstanceFollowRedirects(false); // don't rely on native redirection support
145	conn.setConnectTimeout(req.timeout());
146	conn.setReadTimeout(req.timeout());
147	if (req.method() == Method.POST)
148	conn.setDoOutput(true);
149	if (req.cookies().size() > 0)
150	conn.addRequestProperty("Cookie", getRequestCookieString(req));
151	for (Map.Entry<String, String> header : req.headers().entrySet()) {
152	conn.addRequestProperty(header.getKey(), header.getValue());
153	}
154	return conn;
155	}
156	private static String getRequestCookieString(Connection.Request req) {
157	StringBuilder sb = new StringBuilder();
158	boolean first = true;
159	for (Map.Entry<String, String> cookie : req.cookies().entrySet()) {
160	if (!first)
161	sb.append("; ");
162	else
163	first = false;
164	sb.append(cookie.getKey()).append('=').append(cookie.getValue());
165	// todo: spec says only ascii, no escaping / encoding defined. validate on set? or escape somehow here?
166	}
167	return sb.toString();
168	}
169
170	private static void writePost(Collection<Connection.KeyVal> data, OutputStream outputStream) throws IOException {
171	OutputStreamWriter w = new OutputStreamWriter(outputStream, "UTF-8");
172	boolean first = true;
173	for (Connection.KeyVal keyVal : data) {
174	if (!first)
175	w.append('&');
176	else
177	first = false;
178
179	w.write(URLEncoder.encode(keyVal.key(), "UTF-8"));
180	w.write('=');
181	w.write(URLEncoder.encode(keyVal.value(), "UTF-8"));
182	}
183	w.close();
184	}
185	}