traininfoservice/banedk/CompareStations.java

package dk.thoerup.traininfoservice.banedk;

import java.io.IOException;
import java.io.PrintWriter;
import java.net.URL;
import java.net.URLEncoder;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;

import javax.servlet.ServletException;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import dk.thoerup.android.traininfo.common.StationBean;
import dk.thoerup.android.traininfo.common.StationEntry;
import dk.thoerup.circuitbreaker.CircuitBreaker;
import dk.thoerup.circuitbreaker.CircuitBreakerManager;
import dk.thoerup.traininfoservice.db.StationDAO;

import dk.thoerup.genericjavautils.TimeoutMap;


@WebServlet(urlPatterns={"/CompareStations"})
public class CompareStations extends HttpServlet {
        private static final long serialVersionUID = 1L;

        Map<String,String> cache = new TimeoutMap<String,String>(600 * 1000);
       
        protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
                try {
                
                        String data = getData(request);

                        PrintWriter out = response.getWriter();
                        out.print(data);

                        System.out.println("CompareStations completed");
                } catch (Exception e) {
                        throw new ServletException(e);
                }
        }

        String getData(HttpServletRequest request) throws Exception {
                CircuitBreaker cb = CircuitBreakerManager.getManager().getCircuitBreaker("banedk");
                
                JsoupInvocation jsoup = new JsoupInvocation(new URL("http://trafikinfo.bane.dk/Trafikinformation/Stationsliste"), 5000);
                
                Map<String,String> banedkStations = new TreeMap<String,String>();
                Set<String> dbStations = new TreeSet<String>();
        
                StringBuilder sb = new StringBuilder(); 
                sb.append("<html>");
                sb.append("<head></head>");
                sb.append("<body>");
        
                StationDAO dao = new StationDAO();
                
                try {
                        Document doc = (Document) cb.invoke(jsoup);
                        
                        Elements tables = doc.getElementsByClass("Oversigt");
                        
                        for(Element e : tables) {
                                if (e.tagName().equals("table") ){
                                        Elements links = e.getElementsByTag("a");
                                        for (Element link : links) {                                            
                                                banedkStations.put( link.text(), link.attr("href") );
                                        }
                                }
                        }

                        sb.append("<h2>Active stations in DB</h2>\n");
                        sb.append("<ul>");

                        StationBean bean = dao.dumpAll();
                        for (StationEntry station : bean.entries) {
                                if (station.isRegional() || station.isStrain()) {
                                        if (station.isRegional() ) {            
                                                String id = URLEncoder.encode(station.getRegional(), "ISO-8859-1");
                                                String uri = "/Trafikinformation/AfgangAnkomst/Afgang/" + id + "/Fjerntog";
                                        
                                                String data = hasData(uri, true);

                                                sb.append( "<li><a target='_blank' href='http://trafikinfo.bane.dk" + uri + "'>" + station.getName() + "</a>&nbsp;&nbsp;" + data + "</li>\n" );
                                        }
                                        if (station.isStrain() ) {              
                                                String id = URLEncoder.encode(station.getStrain(), "ISO-8859-1");
                                                String uri = "/Trafikinformation/AfgangAnkomst/Afgang/" + id + "/S-Tog";
                                        
                                                String data = hasData(uri, true);

                                                sb.append( "<li><a target='_blank' href='http://trafikinfo.bane.dk" + uri + "'>" + station.getName() + "</a>&nbsp;&nbsp;" + data + "</li>\n" );
                                        }

                                        dbStations.add(station.getName());
                                }
                        }
                        sb.append("</ul>\n");
                        
                } catch (Exception e) {
                        throw new ServletException(e);
                }
                
                
                sb.append("<h2>stations on website not in db</h2>\n");
                sb.append("<ul>");
                Set<String> tmpDbStations = new TreeSet<String>( dbStations );
                Set<String> tmpBanedkStations = new TreeSet<String>( banedkStations.keySet()) ;
                tmpBanedkStations.removeAll(tmpDbStations);

                
                for(String s : tmpBanedkStations) {
                        String uri = banedkStations.get(s);
                        String disabled = "";
                        String data = hasData(uri, false);
                        
                        try {
                                boolean tmpdisabled = dao.hasDisabledStation(s);
                                if (tmpdisabled == true)
                                        disabled = " - disabled";
                        } catch (Exception e) {
                                throw new ServletException(e);
                        }
                         
                        sb.append( "<li><a target='_blank' href='http://trafikinfo.bane.dk" + uri + "'>" + s + "</a>&nbsp;&nbsp;" + data + "" + disabled + "</li>\n" );
                }                               
                sb.append("</ul>");

                
                sb.append("<h2>stations in db not on website </h2>");
                sb.append("<ul>");
                tmpDbStations = new TreeSet<String>( dbStations );
                tmpBanedkStations = new TreeSet<String>( banedkStations.keySet() ) ;
                tmpDbStations.removeAll(tmpBanedkStations);
                for(String s : tmpDbStations) {
                        sb.append( "<li>" + s + "</li>\n" );
                }                               
                sb.append("</ul>");

                sb.append("</body></html>\n");
                
                return sb.toString();   
                
        }
        
        String hasData(String uri, boolean requiredata) {
                String key = uri + requiredata;

                String data = cache.get(key);
                if (data == null) {
                
                        data = "<font color='yellow'>unknown</font>";
                        try {
                                data = hasDataWorker(uri,requiredata);
                                cache.put(key,data);
                        } catch (Exception e) {
                                System.out.println( e.getMessage() );
                        }
                }
                return data;
        }

        String hasDataWorker(String uri, boolean requiredata) throws Exception {
                String returnVal = ""; 
                
                CircuitBreaker cb = CircuitBreakerManager.getManager().getCircuitBreaker("banedk");
                        
                JsoupInvocation jsoup = new JsoupInvocation(new URL("http://trafikinfo.bane.dk" + uri), 2000);
                        
                Document doc = (Document) cb.invoke(jsoup);
                        
                Element tabel = doc.getElementById("afgangtabel");
                if ( tabel != null) {
                        if (requiredata == true) {
                                returnVal = "";
                        } else {        
                                returnVal = "<font color='red'>Data!</font>"; 
                        }
                } else {
                        if (requiredata == true) {
                                returnVal = "<font color='red'>No data</font>"; 
                        } else {
                                returnVal =  "";
                        }
                }
                        
                
                return returnVal;
        }


}
1	package dk.thoerup.traininfoservice.banedk;
2
3	import java.io.IOException;
4	import java.io.PrintWriter;
5	import java.net.URL;
6	import java.net.URLEncoder;
7	import java.util.Map;
8	import java.util.Set;
9	import java.util.TreeMap;
10	import java.util.TreeSet;
11
12	import javax.servlet.ServletException;
13	import javax.servlet.annotation.WebServlet;
14	import javax.servlet.http.HttpServlet;
15	import javax.servlet.http.HttpServletRequest;
16	import javax.servlet.http.HttpServletResponse;
17
18	import org.jsoup.nodes.Document;
19	import org.jsoup.nodes.Element;
20	import org.jsoup.select.Elements;
21
22	import dk.thoerup.android.traininfo.common.StationBean;
23	import dk.thoerup.android.traininfo.common.StationEntry;
24	import dk.thoerup.circuitbreaker.CircuitBreaker;
25	import dk.thoerup.circuitbreaker.CircuitBreakerManager;
26	import dk.thoerup.traininfoservice.db.StationDAO;
27
28	import dk.thoerup.genericjavautils.TimeoutMap;
29
30
31	@WebServlet(urlPatterns={"/CompareStations"})
32	public class CompareStations extends HttpServlet {
33	private static final long serialVersionUID = 1L;
34
35	Map<String,String> cache = new TimeoutMap<String,String>(600 * 1000);
36
37	protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
38	try {
39
40	String data = getData(request);
41
42	PrintWriter out = response.getWriter();
43	out.print(data);
44
45	System.out.println("CompareStations completed");
46	} catch (Exception e) {
47	throw new ServletException(e);
48	}
49	}
50
51	String getData(HttpServletRequest request) throws Exception {
52	CircuitBreaker cb = CircuitBreakerManager.getManager().getCircuitBreaker("banedk");
53
54	JsoupInvocation jsoup = new JsoupInvocation(new URL("http://trafikinfo.bane.dk/Trafikinformation/Stationsliste"), 5000);
55
56	Map<String,String> banedkStations = new TreeMap<String,String>();
57	Set<String> dbStations = new TreeSet<String>();
58
59	StringBuilder sb = new StringBuilder();
60	sb.append("<html>");
61	sb.append("<head></head>");
62	sb.append("<body>");
63
64	StationDAO dao = new StationDAO();
65
66	try {
67	Document doc = (Document) cb.invoke(jsoup);
68
69	Elements tables = doc.getElementsByClass("Oversigt");
70
71	for(Element e : tables) {
72	if (e.tagName().equals("table") ){
73	Elements links = e.getElementsByTag("a");
74	for (Element link : links) {
75	banedkStations.put( link.text(), link.attr("href") );
76	}
77	}
78	}
79
80	sb.append("<h2>Active stations in DB</h2>\n");
81	sb.append("<ul>");
82
83	StationBean bean = dao.dumpAll();
84	for (StationEntry station : bean.entries) {
85	if (station.isRegional() \|\| station.isStrain()) {
86	if (station.isRegional() ) {
87	String id = URLEncoder.encode(station.getRegional(), "ISO-8859-1");
88	String uri = "/Trafikinformation/AfgangAnkomst/Afgang/" + id + "/Fjerntog";
89
90	String data = hasData(uri, true);
91
92	sb.append( "<li><a target='_blank' href='http://trafikinfo.bane.dk" + uri + "'>" + station.getName() + "</a>  " + data + "</li>\n" );
93	}
94	if (station.isStrain() ) {
95	String id = URLEncoder.encode(station.getStrain(), "ISO-8859-1");
96	String uri = "/Trafikinformation/AfgangAnkomst/Afgang/" + id + "/S-Tog";
97
98	String data = hasData(uri, true);
99
100	sb.append( "<li><a target='_blank' href='http://trafikinfo.bane.dk" + uri + "'>" + station.getName() + "</a>  " + data + "</li>\n" );
101	}
102
103	dbStations.add(station.getName());
104	}
105	}
106	sb.append("</ul>\n");
107
108	} catch (Exception e) {
109	throw new ServletException(e);
110	}
111
112
113
114	sb.append("<h2>stations on website not in db</h2>\n");
115	sb.append("<ul>");
116	Set<String> tmpDbStations = new TreeSet<String>( dbStations );
117	Set<String> tmpBanedkStations = new TreeSet<String>( banedkStations.keySet()) ;
118	tmpBanedkStations.removeAll(tmpDbStations);
119
120
121
122	for(String s : tmpBanedkStations) {
123	String uri = banedkStations.get(s);
124	String disabled = "";
125	String data = hasData(uri, false);
126
127	try {
128	boolean tmpdisabled = dao.hasDisabledStation(s);
129	if (tmpdisabled == true)
130	disabled = " - disabled";
131	} catch (Exception e) {
132	throw new ServletException(e);
133	}
134
135	sb.append( "<li><a target='_blank' href='http://trafikinfo.bane.dk" + uri + "'>" + s + "</a>  " + data + "" + disabled + "</li>\n" );
136	}
137	sb.append("</ul>");
138
139
140	sb.append("<h2>stations in db not on website </h2>");
141	sb.append("<ul>");
142	tmpDbStations = new TreeSet<String>( dbStations );
143	tmpBanedkStations = new TreeSet<String>( banedkStations.keySet() ) ;
144	tmpDbStations.removeAll(tmpBanedkStations);
145	for(String s : tmpDbStations) {
146	sb.append( "<li>" + s + "</li>\n" );
147	}
148	sb.append("</ul>");
149
150	sb.append("</body></html>\n");
151
152	return sb.toString();
153
154	}
155
156	String hasData(String uri, boolean requiredata) {
157	String key = uri + requiredata;
158
159	String data = cache.get(key);
160	if (data == null) {
161
162	data = "<font color='yellow'>unknown</font>";
163	try {
164	data = hasDataWorker(uri,requiredata);
165	cache.put(key,data);
166	} catch (Exception e) {
167	System.out.println( e.getMessage() );
168	}
169	}
170	return data;
171	}
172
173	String hasDataWorker(String uri, boolean requiredata) throws Exception {
174	String returnVal = "";
175
176	CircuitBreaker cb = CircuitBreakerManager.getManager().getCircuitBreaker("banedk");
177
178	JsoupInvocation jsoup = new JsoupInvocation(new URL("http://trafikinfo.bane.dk" + uri), 2000);
179
180	Document doc = (Document) cb.invoke(jsoup);
181
182	Element tabel = doc.getElementById("afgangtabel");
183	if ( tabel != null) {
184	if (requiredata == true) {
185	returnVal = "";
186	} else {
187	returnVal = "<font color='red'>Data!</font>";
188	}
189	} else {
190	if (requiredata == true) {
191	returnVal = "<font color='red'>No data</font>";
192	} else {
193	returnVal = "";
194	}
195	}
196
197
198	return returnVal;
199	}
200
201
202	}