thoerup/spejdernetscraper/OKMobilSaldo.java

package dk.thoerup.spejdernetscraper;

import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.net.HttpURLConnection;
import java.net.URLEncoder;
import java.util.Collection;
import java.util.List;
import java.util.Map;

import org.jsoup.Connection;
import org.jsoup.Connection.Method;
import org.jsoup.Connection.Request;
import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

public class OKMobilSaldo {

        final String mobil = "71750247";
        final String password = "gc93ehcv";

        final String authKey = ".ASPXAUTH"; 

        String chromeUserAgent = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36";


        public String hentSaldo() {
                try {
                        String frontpageUrl = "https://www.ok-mobil.dk/";
                        String loginUrl = "https://www.ok-mobil.dk/log-paa";
                        String accountUrl = "https://www.ok-mobil.dk/min-konto";

                        Response  res = Jsoup.connect(frontpageUrl)
                                        .userAgent(chromeUserAgent)
                                        .execute();

                        Map<String, String> cookies = res.cookies();


                        /* da OK -mobil sender AUTH cookie to gange skal der lidt ekstra til for at finde den rigtige */
                        Request req = Jsoup
                                        .connect( loginUrl )
                                        .userAgent(chromeUserAgent)
                                        .referrer(frontpageUrl)
                                        .data("phoneNumber", mobil, 
                                                        "password", password
                                                        )
                                                        .method(Method.POST)
                                                        .cookies(cookies)
                                                        .request();


                        HttpURLConnection conn = createConnection(req);
                        conn.connect();
                        if (req.method() == Connection.Method.POST)
                                writePost(req.data(), conn.getOutputStream());
                        
                        int status = conn.getResponseCode();
                        
                        String auth = getAuthHeader(conn.getHeaderFields() );
                        
                        System.out.println("AUTH: " + auth);
                        
                        cookies.put(authKey, auth);
                        /* slut på OK cookie */
                        
        
                        Document accountPage = Jsoup.connect(accountUrl)
                                        .cookies(cookies)
                                        .userAgent(chromeUserAgent)
                                        .get();
                        

                        Elements infoElems = accountPage.getElementsByClass("cashBalance");

                        System.out.println(accountPage.text() );

                        return infoElems.text();

                } catch (Exception e) {
                        return "Fejl: " + e.getMessage();
                }
        }
        
        
        // My helper functions
        String mapToString(Map<String,String> map) {
                StringBuilder sb = new StringBuilder();
                for (String key : map.keySet()) {
                        sb.append(key).append(": ").append(map.get(key)).append("\n");
                }
                return sb.toString();
        }
        
        String maplistToString(Map<String,List<String>> map) {
                StringBuilder sb = new StringBuilder();
                for (String key : map.keySet()) {
                        List<String> vals = map.get(key);
                        for (String val : vals) {
                                sb.append(key).append(": ").append(val).append("\n");
                        }
                }
                return sb.toString();
        }
        
        String getAuthHeader(Map<String,List<String>> map) {
                
                for (String key : map.keySet()) {
                        System.out.println("Key:" + key);
                        
                        if (key == null) //Warning key may be null 
                                continue;
                        
                        if (key.equalsIgnoreCase("Set-Cookie")) {
                                List<String> vals = map.get(key);
                                for (String val : vals) {                               
                                        String parts[] = val.split(";");
                                        String baseCookie[] = parts[0].split("=");
                                        
                                        String cookieName = baseCookie[0];
                                        System.out.println("Cookie:" + cookieName);
                                        
                                        if (cookieName.equalsIgnoreCase(authKey)) {
                                                if (baseCookie.length==2 && baseCookie[1].length() > 10)
                                                        return baseCookie[1];
                                        }
                                }
                        }
                }
                return null;
        }
        
        
        //Snipped from org.jsoup.helper.HttpConnection.java

        private static HttpURLConnection createConnection(Connection.Request req) throws IOException {
                HttpURLConnection conn = (HttpURLConnection) req.url().openConnection();
                conn.setRequestMethod(req.method().name());
                conn.setInstanceFollowRedirects(false); // don't rely on native redirection support
                conn.setConnectTimeout(req.timeout());
                conn.setReadTimeout(req.timeout());
                if (req.method() == Method.POST)
                        conn.setDoOutput(true);
                if (req.cookies().size() > 0)
                        conn.addRequestProperty("Cookie", getRequestCookieString(req));
                for (Map.Entry<String, String> header : req.headers().entrySet()) {
                        conn.addRequestProperty(header.getKey(), header.getValue());
                }
                return conn;
        }
        private static String getRequestCookieString(Connection.Request req) {
                StringBuilder sb = new StringBuilder();
                boolean first = true;
                for (Map.Entry<String, String> cookie : req.cookies().entrySet()) {
                        if (!first)
                                sb.append("; ");
                        else
                                first = false;
                        sb.append(cookie.getKey()).append('=').append(cookie.getValue());
                        // todo: spec says only ascii, no escaping / encoding defined. validate on set? or escape somehow here?
                }
                return sb.toString();
        }

        private static void writePost(Collection<Connection.KeyVal> data, OutputStream outputStream) throws IOException {
                OutputStreamWriter w = new OutputStreamWriter(outputStream, "UTF-8");
                boolean first = true;
                for (Connection.KeyVal keyVal : data) {
                        if (!first) 
                                w.append('&');
                        else
                                first = false;

                        w.write(URLEncoder.encode(keyVal.key(), "UTF-8"));
                        w.write('=');
                        w.write(URLEncoder.encode(keyVal.value(), "UTF-8"));
                }
                w.close();
        }
}
1	torben	2044	package dk.thoerup.spejdernetscraper;
2
3			import java.io.IOException;
4			import java.io.OutputStream;
5			import java.io.OutputStreamWriter;
6			import java.net.HttpURLConnection;
7			import java.net.URLEncoder;
8			import java.util.Collection;
9			import java.util.List;
10			import java.util.Map;
11
12			import org.jsoup.Connection;
13			import org.jsoup.Connection.Method;
14			import org.jsoup.Connection.Request;
15			import org.jsoup.Connection.Response;
16			import org.jsoup.Jsoup;
17			import org.jsoup.nodes.Document;
18			import org.jsoup.select.Elements;
19
20			public class OKMobilSaldo {
21
22			final String mobil = "71750247";
23			final String password = "gc93ehcv";
24
25			final String authKey = ".ASPXAUTH";
26
27			String chromeUserAgent = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36";
28
29
30
31			public String hentSaldo() {
32			try {
33			String frontpageUrl = "https://www.ok-mobil.dk/";
34			String loginUrl = "https://www.ok-mobil.dk/log-paa";
35			String accountUrl = "https://www.ok-mobil.dk/min-konto";
36
37			Response res = Jsoup.connect(frontpageUrl)
38			.userAgent(chromeUserAgent)
39			.execute();
40
41			Map<String, String> cookies = res.cookies();
42
43
44	torben	2066	/* da OK -mobil sender AUTH cookie to gange skal der lidt ekstra til for at finde den rigtige */
45	torben	2044	Request req = Jsoup
46			.connect( loginUrl )
47			.userAgent(chromeUserAgent)
48			.referrer(frontpageUrl)
49			.data("phoneNumber", mobil,
50			"password", password
51			)
52			.method(Method.POST)
53			.cookies(cookies)
54			.request();
55
56
57			HttpURLConnection conn = createConnection(req);
58			conn.connect();
59			if (req.method() == Connection.Method.POST)
60			writePost(req.data(), conn.getOutputStream());
61
62			int status = conn.getResponseCode();
63
64			String auth = getAuthHeader(conn.getHeaderFields() );
65
66			System.out.println("AUTH: " + auth);
67
68			cookies.put(authKey, auth);
69	torben	2066	/* slut på OK cookie */
70	torben	2044
71
72
73			Document accountPage = Jsoup.connect(accountUrl)
74			.cookies(cookies)
75			.userAgent(chromeUserAgent)
76			.get();
77
78
79			Elements infoElems = accountPage.getElementsByClass("cashBalance");
80
81			System.out.println(accountPage.text() );
82
83			return infoElems.text();
84
85			} catch (Exception e) {
86			return "Fejl: " + e.getMessage();
87			}
88			}
89
90
91			// My helper functions
92			String mapToString(Map<String,String> map) {
93			StringBuilder sb = new StringBuilder();
94			for (String key : map.keySet()) {
95			sb.append(key).append(": ").append(map.get(key)).append("\n");
96			}
97			return sb.toString();
98			}
99
100			String maplistToString(Map<String,List<String>> map) {
101			StringBuilder sb = new StringBuilder();
102			for (String key : map.keySet()) {
103			List<String> vals = map.get(key);
104			for (String val : vals) {
105			sb.append(key).append(": ").append(val).append("\n");
106			}
107			}
108			return sb.toString();
109			}
110
111			String getAuthHeader(Map<String,List<String>> map) {
112
113			for (String key : map.keySet()) {
114			System.out.println("Key:" + key);
115
116			if (key == null) //Warning key may be null
117			continue;
118
119			if (key.equalsIgnoreCase("Set-Cookie")) {
120			List<String> vals = map.get(key);
121			for (String val : vals) {
122			String parts[] = val.split(";");
123			String baseCookie[] = parts[0].split("=");
124
125			String cookieName = baseCookie[0];
126			System.out.println("Cookie:" + cookieName);
127
128			if (cookieName.equalsIgnoreCase(authKey)) {
129			if (baseCookie.length==2 && baseCookie[1].length() > 10)
130			return baseCookie[1];
131			}
132			}
133			}
134			}
135			return null;
136			}
137
138
139
140			//Snipped from org.jsoup.helper.HttpConnection.java
141
142			private static HttpURLConnection createConnection(Connection.Request req) throws IOException {
143			HttpURLConnection conn = (HttpURLConnection) req.url().openConnection();
144			conn.setRequestMethod(req.method().name());
145			conn.setInstanceFollowRedirects(false); // don't rely on native redirection support
146			conn.setConnectTimeout(req.timeout());
147			conn.setReadTimeout(req.timeout());
148			if (req.method() == Method.POST)
149			conn.setDoOutput(true);
150			if (req.cookies().size() > 0)
151			conn.addRequestProperty("Cookie", getRequestCookieString(req));
152			for (Map.Entry<String, String> header : req.headers().entrySet()) {
153			conn.addRequestProperty(header.getKey(), header.getValue());
154			}
155			return conn;
156			}
157			private static String getRequestCookieString(Connection.Request req) {
158			StringBuilder sb = new StringBuilder();
159			boolean first = true;
160			for (Map.Entry<String, String> cookie : req.cookies().entrySet()) {
161			if (!first)
162			sb.append("; ");
163			else
164			first = false;
165			sb.append(cookie.getKey()).append('=').append(cookie.getValue());
166			// todo: spec says only ascii, no escaping / encoding defined. validate on set? or escape somehow here?
167			}
168			return sb.toString();
169			}
170
171			private static void writePost(Collection<Connection.KeyVal> data, OutputStream outputStream) throws IOException {
172			OutputStreamWriter w = new OutputStreamWriter(outputStream, "UTF-8");
173			boolean first = true;
174			for (Connection.KeyVal keyVal : data) {
175			if (!first)
176			w.append('&');
177			else
178			first = false;
179
180			w.write(URLEncoder.encode(keyVal.key(), "UTF-8"));
181			w.write('=');
182			w.write(URLEncoder.encode(keyVal.value(), "UTF-8"));
183			}
184			w.close();
185			}
186			}