thoerup/spejdernetscraper/OKMobilSaldo.java

package dk.thoerup.spejdernetscraper;

import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.net.HttpURLConnection;
import java.net.URLEncoder;
import java.util.Collection;
import java.util.List;
import java.util.Map;

import org.jsoup.Connection;
import org.jsoup.Connection.Method;
import org.jsoup.Connection.Request;
import org.jsoup.Connection.Response;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

public class OKMobilSaldo {

        final String mobil = "71750247";
        final String password = "gc93ehcv";

        final String authKey = ".ASPXAUTH"; 

        String chromeUserAgent = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36";


        public String hentSaldo() {
                try {
                        String frontpageUrl = "https://www.ok-mobil.dk/";
                        String loginUrl = "https://www.ok-mobil.dk/log-paa";
                        String accountUrl = "https://www.ok-mobil.dk/min-konto";

                        Response  res = Jsoup.connect(frontpageUrl)
                                        .userAgent(chromeUserAgent)
                                        .execute();

                        Map<String, String> cookies = res.cookies();


                        Request req = Jsoup
                                        .connect( loginUrl )
                                        .userAgent(chromeUserAgent)
                                        .referrer(frontpageUrl)
                                        .data("phoneNumber", mobil, 
                                                        "password", password
                                                        )
                                                        .method(Method.POST)
                                                        .cookies(cookies)
                                                        .request();


                        HttpURLConnection conn = createConnection(req);
                        conn.connect();
                        if (req.method() == Connection.Method.POST)
                                writePost(req.data(), conn.getOutputStream());
                        
                        int status = conn.getResponseCode();
                        
                        String auth = getAuthHeader(conn.getHeaderFields() );
                        
                        System.out.println("AUTH: " + auth);
                        
                        cookies.put(authKey, auth);
                        
        
                        Document accountPage = Jsoup.connect(accountUrl)
                                        .cookies(cookies)
                                        .userAgent(chromeUserAgent)
                                        .get();
                        

                        Elements infoElems = accountPage.getElementsByClass("cashBalance");

                        System.out.println(accountPage.text() );

                        return infoElems.text();

                } catch (Exception e) {
                        return "Fejl: " + e.getMessage();
                }
        }
        
        
        // My helper functions
        String mapToString(Map<String,String> map) {
                StringBuilder sb = new StringBuilder();
                for (String key : map.keySet()) {
                        sb.append(key).append(": ").append(map.get(key)).append("\n");
                }
                return sb.toString();
        }
        
        String maplistToString(Map<String,List<String>> map) {
                StringBuilder sb = new StringBuilder();
                for (String key : map.keySet()) {
                        List<String> vals = map.get(key);
                        for (String val : vals) {
                                sb.append(key).append(": ").append(val).append("\n");
                        }
                }
                return sb.toString();
        }
        
        String getAuthHeader(Map<String,List<String>> map) {
                
                for (String key : map.keySet()) {
                        System.out.println("Key:" + key);
                        
                        if (key == null) //Warning key may be null 
                                continue;
                        
                        if (key.equalsIgnoreCase("Set-Cookie")) {
                                List<String> vals = map.get(key);
                                for (String val : vals) {                               
                                        String parts[] = val.split(";");
                                        String baseCookie[] = parts[0].split("=");
                                        
                                        String cookieName = baseCookie[0];
                                        System.out.println("Cookie:" + cookieName);
                                        
                                        if (cookieName.equalsIgnoreCase(authKey)) {
                                                if (baseCookie.length==2 && baseCookie[1].length() > 10)
                                                        return baseCookie[1];
                                        }
                                }
                        }
                }
                return null;
        }
        
        
        //Snipped from org.jsoup.helper.HttpConnection.java

        private static HttpURLConnection createConnection(Connection.Request req) throws IOException {
                HttpURLConnection conn = (HttpURLConnection) req.url().openConnection();
                conn.setRequestMethod(req.method().name());
                conn.setInstanceFollowRedirects(false); // don't rely on native redirection support
                conn.setConnectTimeout(req.timeout());
                conn.setReadTimeout(req.timeout());
                if (req.method() == Method.POST)
                        conn.setDoOutput(true);
                if (req.cookies().size() > 0)
                        conn.addRequestProperty("Cookie", getRequestCookieString(req));
                for (Map.Entry<String, String> header : req.headers().entrySet()) {
                        conn.addRequestProperty(header.getKey(), header.getValue());
                }
                return conn;
        }
        private static String getRequestCookieString(Connection.Request req) {
                StringBuilder sb = new StringBuilder();
                boolean first = true;
                for (Map.Entry<String, String> cookie : req.cookies().entrySet()) {
                        if (!first)
                                sb.append("; ");
                        else
                                first = false;
                        sb.append(cookie.getKey()).append('=').append(cookie.getValue());
                        // todo: spec says only ascii, no escaping / encoding defined. validate on set? or escape somehow here?
                }
                return sb.toString();
        }

        private static void writePost(Collection<Connection.KeyVal> data, OutputStream outputStream) throws IOException {
                OutputStreamWriter w = new OutputStreamWriter(outputStream, "UTF-8");
                boolean first = true;
                for (Connection.KeyVal keyVal : data) {
                        if (!first) 
                                w.append('&');
                        else
                                first = false;

                        w.write(URLEncoder.encode(keyVal.key(), "UTF-8"));
                        w.write('=');
                        w.write(URLEncoder.encode(keyVal.value(), "UTF-8"));
                }
                w.close();
        }
}
1	torben	2044	package dk.thoerup.spejdernetscraper;
2
3			import java.io.IOException;
4			import java.io.OutputStream;
5			import java.io.OutputStreamWriter;
6			import java.net.HttpURLConnection;
7			import java.net.URLEncoder;
8			import java.util.Collection;
9			import java.util.List;
10			import java.util.Map;
11
12			import org.jsoup.Connection;
13			import org.jsoup.Connection.Method;
14			import org.jsoup.Connection.Request;
15			import org.jsoup.Connection.Response;
16			import org.jsoup.Jsoup;
17			import org.jsoup.nodes.Document;
18			import org.jsoup.select.Elements;
19
20			public class OKMobilSaldo {
21
22			final String mobil = "71750247";
23			final String password = "gc93ehcv";
24
25			final String authKey = ".ASPXAUTH";
26
27			String chromeUserAgent = "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36";
28
29
30
31			public String hentSaldo() {
32			try {
33			String frontpageUrl = "https://www.ok-mobil.dk/";
34			String loginUrl = "https://www.ok-mobil.dk/log-paa";
35			String accountUrl = "https://www.ok-mobil.dk/min-konto";
36
37			Response res = Jsoup.connect(frontpageUrl)
38			.userAgent(chromeUserAgent)
39			.execute();
40
41			Map<String, String> cookies = res.cookies();
42
43
44
45			Request req = Jsoup
46			.connect( loginUrl )
47			.userAgent(chromeUserAgent)
48			.referrer(frontpageUrl)
49			.data("phoneNumber", mobil,
50			"password", password
51			)
52			.method(Method.POST)
53			.cookies(cookies)
54			.request();
55
56
57			HttpURLConnection conn = createConnection(req);
58			conn.connect();
59			if (req.method() == Connection.Method.POST)
60			writePost(req.data(), conn.getOutputStream());
61
62			int status = conn.getResponseCode();
63
64			String auth = getAuthHeader(conn.getHeaderFields() );
65
66			System.out.println("AUTH: " + auth);
67
68			cookies.put(authKey, auth);
69
70
71
72			Document accountPage = Jsoup.connect(accountUrl)
73			.cookies(cookies)
74			.userAgent(chromeUserAgent)
75			.get();
76
77
78			Elements infoElems = accountPage.getElementsByClass("cashBalance");
79
80			System.out.println(accountPage.text() );
81
82			return infoElems.text();
83
84			} catch (Exception e) {
85			return "Fejl: " + e.getMessage();
86			}
87			}
88
89
90			// My helper functions
91			String mapToString(Map<String,String> map) {
92			StringBuilder sb = new StringBuilder();
93			for (String key : map.keySet()) {
94			sb.append(key).append(": ").append(map.get(key)).append("\n");
95			}
96			return sb.toString();
97			}
98
99			String maplistToString(Map<String,List<String>> map) {
100			StringBuilder sb = new StringBuilder();
101			for (String key : map.keySet()) {
102			List<String> vals = map.get(key);
103			for (String val : vals) {
104			sb.append(key).append(": ").append(val).append("\n");
105			}
106			}
107			return sb.toString();
108			}
109
110			String getAuthHeader(Map<String,List<String>> map) {
111
112			for (String key : map.keySet()) {
113			System.out.println("Key:" + key);
114
115			if (key == null) //Warning key may be null
116			continue;
117
118			if (key.equalsIgnoreCase("Set-Cookie")) {
119			List<String> vals = map.get(key);
120			for (String val : vals) {
121			String parts[] = val.split(";");
122			String baseCookie[] = parts[0].split("=");
123
124			String cookieName = baseCookie[0];
125			System.out.println("Cookie:" + cookieName);
126
127			if (cookieName.equalsIgnoreCase(authKey)) {
128			if (baseCookie.length==2 && baseCookie[1].length() > 10)
129			return baseCookie[1];
130			}
131			}
132			}
133			}
134			return null;
135			}
136
137
138
139			//Snipped from org.jsoup.helper.HttpConnection.java
140
141			private static HttpURLConnection createConnection(Connection.Request req) throws IOException {
142			HttpURLConnection conn = (HttpURLConnection) req.url().openConnection();
143			conn.setRequestMethod(req.method().name());
144			conn.setInstanceFollowRedirects(false); // don't rely on native redirection support
145			conn.setConnectTimeout(req.timeout());
146			conn.setReadTimeout(req.timeout());
147			if (req.method() == Method.POST)
148			conn.setDoOutput(true);
149			if (req.cookies().size() > 0)
150			conn.addRequestProperty("Cookie", getRequestCookieString(req));
151			for (Map.Entry<String, String> header : req.headers().entrySet()) {
152			conn.addRequestProperty(header.getKey(), header.getValue());
153			}
154			return conn;
155			}
156			private static String getRequestCookieString(Connection.Request req) {
157			StringBuilder sb = new StringBuilder();
158			boolean first = true;
159			for (Map.Entry<String, String> cookie : req.cookies().entrySet()) {
160			if (!first)
161			sb.append("; ");
162			else
163			first = false;
164			sb.append(cookie.getKey()).append('=').append(cookie.getValue());
165			// todo: spec says only ascii, no escaping / encoding defined. validate on set? or escape somehow here?
166			}
167			return sb.toString();
168			}
169
170			private static void writePost(Collection<Connection.KeyVal> data, OutputStream outputStream) throws IOException {
171			OutputStreamWriter w = new OutputStreamWriter(outputStream, "UTF-8");
172			boolean first = true;
173			for (Connection.KeyVal keyVal : data) {
174			if (!first)
175			w.append('&');
176			else
177			first = false;
178
179			w.write(URLEncoder.encode(keyVal.key(), "UTF-8"));
180			w.write('=');
181			w.write(URLEncoder.encode(keyVal.value(), "UTF-8"));
182			}
183			w.close();
184			}
185			}