thoerup/spejdernetscraper/OKMobilSaldo.java

package dk.thoerup.spejdernetscraper;

import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.net.HttpURLConnection;
import java.net.URLEncoder;
import java.util.Collection;
import java.util.List;
import java.util.Map;

import org.jsoup.Connection;
import org.jsoup.Connection.Method;
import org.jsoup.Connection.Request;
import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

public class OKMobilSaldo {

        final String mobil = "71750247";
        final String password = "gc93ehcv";

        final String authKey = ".ASPXAUTH"; 

        String chromeUserAgent = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36";


        public String hentSaldo() {
                try {
                        String frontpageUrl = "https://www.ok-mobil.dk/";
                        String loginUrl = "https://www.ok-mobil.dk/log-paa";
                        String accountUrl = "https://www.ok-mobil.dk/min-konto";

                        Response  res = Jsoup.connect(frontpageUrl)
                                        .userAgent(chromeUserAgent)
                                        .execute();

                        Map<String, String> cookies = res.cookies();


                        /* da OK -mobil sender AUTH cookie to gange skal der lidt ekstra til for at finde den rigtige */
                        Request req = Jsoup
                                        .connect( loginUrl )
                                        .userAgent(chromeUserAgent)
                                        .referrer(frontpageUrl)
                                        .data("phoneNumber", mobil, 
                                                        "password", password
                                                        )
                                                        .method(Method.POST)
                                                        .cookies(cookies)
                                                        .request();


                        HttpURLConnection conn = createConnection(req);
                        conn.connect();
                        if (req.method() == Connection.Method.POST)
                                writePost(req.data(), conn.getOutputStream());
                        
                        int status = conn.getResponseCode();
                        
                        String auth = getAuthHeader(conn.getHeaderFields() );
                        
                        System.out.println("AUTH: " + auth);
                        
                        cookies.put(authKey, auth);
                        /* slut på OK cookie */
                        
        
                        Document accountPage = Jsoup.connect(accountUrl)
                                        .cookies(cookies)
                                        .userAgent(chromeUserAgent)
                                        .get();
                        

                        Elements infoElems = accountPage.getElementsByClass("cashBalance");

                        System.out.println(accountPage.text() );

                        return infoElems.text();

                } catch (Exception e) {
                        return "Fejl: " + e.getMessage();
                }
        }
        
        
        // My helper functions
        String mapToString(Map<String,String> map) {
                StringBuilder sb = new StringBuilder();
                for (String key : map.keySet()) {
                        sb.append(key).append(": ").append(map.get(key)).append("\n");
                }
                return sb.toString();
        }
        
        String maplistToString(Map<String,List<String>> map) {
                StringBuilder sb = new StringBuilder();
                for (String key : map.keySet()) {
                        List<String> vals = map.get(key);
                        for (String val : vals) {
                                sb.append(key).append(": ").append(val).append("\n");
                        }
                }
                return sb.toString();
        }
        
        String getAuthHeader(Map<String,List<String>> map) {
                
                for (String key : map.keySet()) {
                        System.out.println("Key:" + key);
                        
                        if (key == null) //Warning key may be null 
                                continue;
                        
                        if (key.equalsIgnoreCase("Set-Cookie")) {
                                List<String> vals = map.get(key);
                                for (String val : vals) {                               
                                        String parts[] = val.split(";");
                                        String baseCookie[] = parts[0].split("=");
                                        
                                        String cookieName = baseCookie[0];
                                        System.out.println("Cookie:" + cookieName);
                                        
                                        if (cookieName.equalsIgnoreCase(authKey)) {
                                                if (baseCookie.length==2 && baseCookie[1].length() > 10)
                                                        return baseCookie[1];
                                        }
                                }
                        }
                }
                return null;
        }
        
        
        //Snipped from org.jsoup.helper.HttpConnection.java

        private static HttpURLConnection createConnection(Connection.Request req) throws IOException {
                HttpURLConnection conn = (HttpURLConnection) req.url().openConnection();
                conn.setRequestMethod(req.method().name());
                conn.setInstanceFollowRedirects(false); // don't rely on native redirection support
                conn.setConnectTimeout(req.timeout());
                conn.setReadTimeout(req.timeout());
                if (req.method() == Method.POST)
                        conn.setDoOutput(true);
                if (req.cookies().size() > 0)
                        conn.addRequestProperty("Cookie", getRequestCookieString(req));
                for (Map.Entry<String, String> header : req.headers().entrySet()) {
                        conn.addRequestProperty(header.getKey(), header.getValue());
                }
                return conn;
        }
        private static String getRequestCookieString(Connection.Request req) {
                StringBuilder sb = new StringBuilder();
                boolean first = true;
                for (Map.Entry<String, String> cookie : req.cookies().entrySet()) {
                        if (!first)
                                sb.append("; ");
                        else
                                first = false;
                        sb.append(cookie.getKey()).append('=').append(cookie.getValue());
                        // todo: spec says only ascii, no escaping / encoding defined. validate on set? or escape somehow here?
                }
                return sb.toString();
        }

        private static void writePost(Collection<Connection.KeyVal> data, OutputStream outputStream) throws IOException {
                OutputStreamWriter w = new OutputStreamWriter(outputStream, "UTF-8");
                boolean first = true;
                for (Connection.KeyVal keyVal : data) {
                        if (!first) 
                                w.append('&');
                        else
                                first = false;

                        w.write(URLEncoder.encode(keyVal.key(), "UTF-8"));
                        w.write('=');
                        w.write(URLEncoder.encode(keyVal.value(), "UTF-8"));
                }
                w.close();
        }
}
1	package dk.thoerup.spejdernetscraper;
2
3	import java.io.IOException;
4	import java.io.OutputStream;
5	import java.io.OutputStreamWriter;
6	import java.net.HttpURLConnection;
7	import java.net.URLEncoder;
8	import java.util.Collection;
9	import java.util.List;
10	import java.util.Map;
11
12	import org.jsoup.Connection;
13	import org.jsoup.Connection.Method;
14	import org.jsoup.Connection.Request;
15	import org.jsoup.Connection.Response;
16	import org.jsoup.Jsoup;
17	import org.jsoup.nodes.Document;
18	import org.jsoup.select.Elements;
19
20	public class OKMobilSaldo {
21
22	final String mobil = "71750247";
23	final String password = "gc93ehcv";
24
25	final String authKey = ".ASPXAUTH";
26
27	String chromeUserAgent = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36";
28
29
30
31	public String hentSaldo() {
32	try {
33	String frontpageUrl = "https://www.ok-mobil.dk/";
34	String loginUrl = "https://www.ok-mobil.dk/log-paa";
35	String accountUrl = "https://www.ok-mobil.dk/min-konto";
36
37	Response res = Jsoup.connect(frontpageUrl)
38	.userAgent(chromeUserAgent)
39	.execute();
40
41	Map<String, String> cookies = res.cookies();
42
43
44	/* da OK -mobil sender AUTH cookie to gange skal der lidt ekstra til for at finde den rigtige */
45	Request req = Jsoup
46	.connect( loginUrl )
47	.userAgent(chromeUserAgent)
48	.referrer(frontpageUrl)
49	.data("phoneNumber", mobil,
50	"password", password
51	)
52	.method(Method.POST)
53	.cookies(cookies)
54	.request();
55
56
57	HttpURLConnection conn = createConnection(req);
58	conn.connect();
59	if (req.method() == Connection.Method.POST)
60	writePost(req.data(), conn.getOutputStream());
61
62	int status = conn.getResponseCode();
63
64	String auth = getAuthHeader(conn.getHeaderFields() );
65
66	System.out.println("AUTH: " + auth);
67
68	cookies.put(authKey, auth);
69	/* slut på OK cookie */
70
71
72
73	Document accountPage = Jsoup.connect(accountUrl)
74	.cookies(cookies)
75	.userAgent(chromeUserAgent)
76	.get();
77
78
79	Elements infoElems = accountPage.getElementsByClass("cashBalance");
80
81	System.out.println(accountPage.text() );
82
83	return infoElems.text();
84
85	} catch (Exception e) {
86	return "Fejl: " + e.getMessage();
87	}
88	}
89
90
91	// My helper functions
92	String mapToString(Map<String,String> map) {
93	StringBuilder sb = new StringBuilder();
94	for (String key : map.keySet()) {
95	sb.append(key).append(": ").append(map.get(key)).append("\n");
96	}
97	return sb.toString();
98	}
99
100	String maplistToString(Map<String,List<String>> map) {
101	StringBuilder sb = new StringBuilder();
102	for (String key : map.keySet()) {
103	List<String> vals = map.get(key);
104	for (String val : vals) {
105	sb.append(key).append(": ").append(val).append("\n");
106	}
107	}
108	return sb.toString();
109	}
110
111	String getAuthHeader(Map<String,List<String>> map) {
112
113	for (String key : map.keySet()) {
114	System.out.println("Key:" + key);
115
116	if (key == null) //Warning key may be null
117	continue;
118
119	if (key.equalsIgnoreCase("Set-Cookie")) {
120	List<String> vals = map.get(key);
121	for (String val : vals) {
122	String parts[] = val.split(";");
123	String baseCookie[] = parts[0].split("=");
124
125	String cookieName = baseCookie[0];
126	System.out.println("Cookie:" + cookieName);
127
128	if (cookieName.equalsIgnoreCase(authKey)) {
129	if (baseCookie.length==2 && baseCookie[1].length() > 10)
130	return baseCookie[1];
131	}
132	}
133	}
134	}
135	return null;
136	}
137
138
139
140	//Snipped from org.jsoup.helper.HttpConnection.java
141
142	private static HttpURLConnection createConnection(Connection.Request req) throws IOException {
143	HttpURLConnection conn = (HttpURLConnection) req.url().openConnection();
144	conn.setRequestMethod(req.method().name());
145	conn.setInstanceFollowRedirects(false); // don't rely on native redirection support
146	conn.setConnectTimeout(req.timeout());
147	conn.setReadTimeout(req.timeout());
148	if (req.method() == Method.POST)
149	conn.setDoOutput(true);
150	if (req.cookies().size() > 0)
151	conn.addRequestProperty("Cookie", getRequestCookieString(req));
152	for (Map.Entry<String, String> header : req.headers().entrySet()) {
153	conn.addRequestProperty(header.getKey(), header.getValue());
154	}
155	return conn;
156	}
157	private static String getRequestCookieString(Connection.Request req) {
158	StringBuilder sb = new StringBuilder();
159	boolean first = true;
160	for (Map.Entry<String, String> cookie : req.cookies().entrySet()) {
161	if (!first)
162	sb.append("; ");
163	else
164	first = false;
165	sb.append(cookie.getKey()).append('=').append(cookie.getValue());
166	// todo: spec says only ascii, no escaping / encoding defined. validate on set? or escape somehow here?
167	}
168	return sb.toString();
169	}
170
171	private static void writePost(Collection<Connection.KeyVal> data, OutputStream outputStream) throws IOException {
172	OutputStreamWriter w = new OutputStreamWriter(outputStream, "UTF-8");
173	boolean first = true;
174	for (Connection.KeyVal keyVal : data) {
175	if (!first)
176	w.append('&');
177	else
178	first = false;
179
180	w.write(URLEncoder.encode(keyVal.key(), "UTF-8"));
181	w.write('=');
182	w.write(URLEncoder.encode(keyVal.value(), "UTF-8"));
183	}
184	w.close();
185	}
186	}