traininfoservice/banedk/CompareStations.java

package dk.thoerup.traininfoservice.banedk;

import java.io.IOException;
import java.io.PrintWriter;
import java.net.URL;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;

import javax.servlet.ServletException;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import dk.thoerup.android.traininfo.common.StationBean;
import dk.thoerup.android.traininfo.common.StationEntry;
import dk.thoerup.circuitbreaker.CircuitBreaker;
import dk.thoerup.circuitbreaker.CircuitBreakerManager;
import dk.thoerup.traininfoservice.db.StationDAO;

import dk.thoerup.genericjavautils.TimeoutMap;


@WebServlet(urlPatterns={"/CompareStations"})
public class CompareStations extends HttpServlet {
        private static final long serialVersionUID = 1L;

        Map<String,String> cache = new TimeoutMap<String,String>(600 * 1000);
       
        protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
                try {
                        String data = "";
                        String uri = request.getParameter("uri");

                        if (uri != null) {
                                boolean require = Boolean.parseBoolean( request.getParameter("requiredata") );
                                data = hasData(uri, require);
                        } else {
                                data = getData(request);
                        }

                        PrintWriter out = response.getWriter();
                        out.print(data);

                } catch (Exception e) {
                        throw new ServletException(e);
                }
        }

        String getData(HttpServletRequest request) throws Exception {
                CircuitBreaker cb = CircuitBreakerManager.getManager().getCircuitBreaker("banedk");
                
                JsoupInvocation jsoup = new JsoupInvocation(new URL("http://trafikinfo.bane.dk/Trafikinformation/Stationsliste"), 5000);
                
                Map<String,String> banedkStations = new TreeMap<String,String>();
                Set<String> dbStations = new TreeSet<String>();
        
                StringBuilder sb = new StringBuilder(); 
                sb.append("<html>");
                sb.append("<head>");
                sb.append("<script type='text/javascript' src='https://ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js'></script>\n");
                sb.append("<script type='text/javascript' src='compareloader.js'></script>\n");

                sb.append("</head>");
                sb.append("<body>");
        
                StationDAO dao = new StationDAO();
                
                try {
                        Document doc = (Document) cb.invoke(jsoup);
                        
                        Elements tables = doc.getElementsByClass("Oversigt");
                        
                        for(Element e : tables) {
                                if (e.tagName().equals("table") ){
                                        Elements links = e.getElementsByTag("a");
                                        for (Element link : links) {                                            
                                                banedkStations.put( link.text(), link.attr("href") );
                                        }
                                }
                        }

                        sb.append("<h2>Active stations in DB</h2>\n");
                        sb.append("<ul>");

                        StationBean bean = dao.dumpAll();
                        for (StationEntry station : bean.entries) {                             
                                if (station.isRegional() || station.isStrain()) {
                                        String href = banedkStations.get(station.getName() );
                                        String hrefCode = "null";
                                        if ( href != null) {
                                                String hrefParts[] = href.split("/");
                                                hrefCode = URLDecoder.decode(hrefParts[4], "UTF-8");
                                        }
                                        
                                        String id = "";
                                        String uri = "";
                                        if (station.isRegional() ) {            
                                                id = station.getRegional();
                                                uri = "/Trafikinformation/AfgangAnkomst/Afgang/" + id + "/Fjerntog";
                                        }
                                        if (station.isStrain() ) {              
                                                id = station.getStrain();
                                                uri = "/Trafikinformation/AfgangAnkomst/Afgang/" + id + "/S-Tog";
                                        }
                                        
                                        String idMismatch = "";
                                        if (!hrefCode.equals(id) ) {
                                                idMismatch = " ID Mismatch (DB="+id+ ", BaneDK=" + hrefCode + ")";
                                        }
                                                
                                        
                                        sb.append( "<li><a target='_blank' href='http://trafikinfo.bane.dk" + uri + "'>" + station.getName() + "</a> <span class='uri' requiredata='true' uri='" + uri + "'></span> " + idMismatch + "</li>\n" );

                                        dbStations.add(station.getName());                                      
                                }
                                

                        }
                        sb.append("</ul>\n");
                        
                } catch (Exception e) {
                        throw new ServletException(e);
                }
                
                
                sb.append("<h2>stations on website not in db</h2>\n");
                sb.append("<ul>");
                Set<String> tmpDbStations = new TreeSet<String>( dbStations );
                Set<String> tmpBanedkStations = new TreeSet<String>( banedkStations.keySet()) ;
                tmpBanedkStations.removeAll(tmpDbStations);

                
                for(String s : tmpBanedkStations) {
                        String uri = banedkStations.get(s);
                        String disabled = "";
                        //String data = hasData(uri, false);
                        
                        try {
                                boolean tmpdisabled = dao.hasDisabledStation(s);
                                if (tmpdisabled == true)
                                        disabled = " - disabled";
                        } catch (Exception e) {
                                throw new ServletException(e);
                        }
                         
                        sb.append( "<li><a target='_blank' href='http://trafikinfo.bane.dk" + uri + "'>" + s + "</a>&nbsp;<span class='uri' requiredata='false' uri='" + uri + "'></span>" + disabled + "</li>\n" );
                }                               
                sb.append("</ul>");

                
                sb.append("<h2>stations in db not on website </h2>");
                sb.append("<ul>");
                tmpDbStations = new TreeSet<String>( dbStations );
                tmpBanedkStations = new TreeSet<String>( banedkStations.keySet() ) ;
                tmpDbStations.removeAll(tmpBanedkStations);
                for(String s : tmpDbStations) {
                        sb.append( "<li>" + s + "</li>\n" );
                }                               
                sb.append("</ul>");
                sb.append("<font size='1' id='antal'></font>");
                sb.append("</body></html>\n");
                
                return sb.toString();   
                
        }
        
        String hasData(String uri, boolean requiredata) {
                String key = uri + requiredata;

                String data = cache.get(key);
                if (data == null) {
                
                        data = "<font color='yellow'>unknown</font>";
                        try {
                                data = hasDataWorker(uri,requiredata);
                                cache.put(key,data);
                        } catch (Exception e) {
                                System.out.println( e.getMessage() );
                        }
                }
                return data;
        }

        String hasDataWorker(String uri, boolean requiredata) throws Exception {
                String returnVal = ""; 
                
                String parts[] = uri.split("\\/");
                String id = URLEncoder.encode(parts[4], "ISO-8859-1");
                String uri2 = "/TrafikInformation/AfgangAnkomst/Afgang/" + id + "/" + parts[5];

                CircuitBreaker cb = CircuitBreakerManager.getManager().getCircuitBreaker("banedk");
                        
                JsoupInvocation jsoup = new JsoupInvocation(new URL("http://trafikinfo.bane.dk" + uri2), 2000);
                        
                Document doc = (Document) cb.invoke(jsoup);
                        
                Element tabel = doc.getElementById("afgangtabel");
                if ( tabel != null) {
                        if (requiredata == true) {
                                returnVal = "";
                        } else {        
                                returnVal = "<font color='red'>Data!</font>"; 
                        }
                } else {
                        if (requiredata == true) {
                                returnVal = "<font color='red'>No data</font>"; 
                        } else {
                                returnVal =  "";
                        }
                }
                        
                
                return returnVal;
        }


}
1	package dk.thoerup.traininfoservice.banedk;
2
3	import java.io.IOException;
4	import java.io.PrintWriter;
5	import java.net.URL;
6	import java.net.URLDecoder;
7	import java.net.URLEncoder;
8	import java.util.Map;
9	import java.util.Set;
10	import java.util.TreeMap;
11	import java.util.TreeSet;
12
13	import javax.servlet.ServletException;
14	import javax.servlet.annotation.WebServlet;
15	import javax.servlet.http.HttpServlet;
16	import javax.servlet.http.HttpServletRequest;
17	import javax.servlet.http.HttpServletResponse;
18
19	import org.jsoup.nodes.Document;
20	import org.jsoup.nodes.Element;
21	import org.jsoup.select.Elements;
22
23	import dk.thoerup.android.traininfo.common.StationBean;
24	import dk.thoerup.android.traininfo.common.StationEntry;
25	import dk.thoerup.circuitbreaker.CircuitBreaker;
26	import dk.thoerup.circuitbreaker.CircuitBreakerManager;
27	import dk.thoerup.traininfoservice.db.StationDAO;
28
29	import dk.thoerup.genericjavautils.TimeoutMap;
30
31
32	@WebServlet(urlPatterns={"/CompareStations"})
33	public class CompareStations extends HttpServlet {
34	private static final long serialVersionUID = 1L;
35
36	Map<String,String> cache = new TimeoutMap<String,String>(600 * 1000);
37
38	protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
39	try {
40	String data = "";
41	String uri = request.getParameter("uri");
42
43	if (uri != null) {
44	boolean require = Boolean.parseBoolean( request.getParameter("requiredata") );
45	data = hasData(uri, require);
46	} else {
47	data = getData(request);
48	}
49
50	PrintWriter out = response.getWriter();
51	out.print(data);
52
53	} catch (Exception e) {
54	throw new ServletException(e);
55	}
56	}
57
58	String getData(HttpServletRequest request) throws Exception {
59	CircuitBreaker cb = CircuitBreakerManager.getManager().getCircuitBreaker("banedk");
60
61	JsoupInvocation jsoup = new JsoupInvocation(new URL("http://trafikinfo.bane.dk/Trafikinformation/Stationsliste"), 5000);
62
63	Map<String,String> banedkStations = new TreeMap<String,String>();
64	Set<String> dbStations = new TreeSet<String>();
65
66	StringBuilder sb = new StringBuilder();
67	sb.append("<html>");
68	sb.append("<head>");
69	sb.append("<script type='text/javascript' src='https://ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js'></script>\n");
70	sb.append("<script type='text/javascript' src='compareloader.js'></script>\n");
71
72	sb.append("</head>");
73	sb.append("<body>");
74
75	StationDAO dao = new StationDAO();
76
77	try {
78	Document doc = (Document) cb.invoke(jsoup);
79
80	Elements tables = doc.getElementsByClass("Oversigt");
81
82	for(Element e : tables) {
83	if (e.tagName().equals("table") ){
84	Elements links = e.getElementsByTag("a");
85	for (Element link : links) {
86	banedkStations.put( link.text(), link.attr("href") );
87	}
88	}
89	}
90
91	sb.append("<h2>Active stations in DB</h2>\n");
92	sb.append("<ul>");
93
94	StationBean bean = dao.dumpAll();
95	for (StationEntry station : bean.entries) {
96	if (station.isRegional() \|\| station.isStrain()) {
97	String href = banedkStations.get(station.getName() );
98	String hrefCode = "null";
99	if ( href != null) {
100	String hrefParts[] = href.split("/");
101	hrefCode = URLDecoder.decode(hrefParts[4], "UTF-8");
102	}
103
104	String id = "";
105	String uri = "";
106	if (station.isRegional() ) {
107	id = station.getRegional();
108	uri = "/Trafikinformation/AfgangAnkomst/Afgang/" + id + "/Fjerntog";
109	}
110	if (station.isStrain() ) {
111	id = station.getStrain();
112	uri = "/Trafikinformation/AfgangAnkomst/Afgang/" + id + "/S-Tog";
113	}
114
115	String idMismatch = "";
116	if (!hrefCode.equals(id) ) {
117	idMismatch = " ID Mismatch (DB="+id+ ", BaneDK=" + hrefCode + ")";
118	}
119
120
121	sb.append( "<li><a target='_blank' href='http://trafikinfo.bane.dk" + uri + "'>" + station.getName() + "</a> <span class='uri' requiredata='true' uri='" + uri + "'></span> " + idMismatch + "</li>\n" );
122
123	dbStations.add(station.getName());
124	}
125
126
127	}
128	sb.append("</ul>\n");
129
130	} catch (Exception e) {
131	throw new ServletException(e);
132	}
133
134
135
136	sb.append("<h2>stations on website not in db</h2>\n");
137	sb.append("<ul>");
138	Set<String> tmpDbStations = new TreeSet<String>( dbStations );
139	Set<String> tmpBanedkStations = new TreeSet<String>( banedkStations.keySet()) ;
140	tmpBanedkStations.removeAll(tmpDbStations);
141
142
143
144	for(String s : tmpBanedkStations) {
145	String uri = banedkStations.get(s);
146	String disabled = "";
147	//String data = hasData(uri, false);
148
149	try {
150	boolean tmpdisabled = dao.hasDisabledStation(s);
151	if (tmpdisabled == true)
152	disabled = " - disabled";
153	} catch (Exception e) {
154	throw new ServletException(e);
155	}
156
157	sb.append( "<li><a target='_blank' href='http://trafikinfo.bane.dk" + uri + "'>" + s + "</a> <span class='uri' requiredata='false' uri='" + uri + "'></span>" + disabled + "</li>\n" );
158	}
159	sb.append("</ul>");
160
161
162	sb.append("<h2>stations in db not on website </h2>");
163	sb.append("<ul>");
164	tmpDbStations = new TreeSet<String>( dbStations );
165	tmpBanedkStations = new TreeSet<String>( banedkStations.keySet() ) ;
166	tmpDbStations.removeAll(tmpBanedkStations);
167	for(String s : tmpDbStations) {
168	sb.append( "<li>" + s + "</li>\n" );
169	}
170	sb.append("</ul>");
171	sb.append("<font size='1' id='antal'></font>");
172	sb.append("</body></html>\n");
173
174	return sb.toString();
175
176	}
177
178	String hasData(String uri, boolean requiredata) {
179	String key = uri + requiredata;
180
181	String data = cache.get(key);
182	if (data == null) {
183
184	data = "<font color='yellow'>unknown</font>";
185	try {
186	data = hasDataWorker(uri,requiredata);
187	cache.put(key,data);
188	} catch (Exception e) {
189	System.out.println( e.getMessage() );
190	}
191	}
192	return data;
193	}
194
195	String hasDataWorker(String uri, boolean requiredata) throws Exception {
196	String returnVal = "";
197
198	String parts[] = uri.split("\\/");
199	String id = URLEncoder.encode(parts[4], "ISO-8859-1");
200	String uri2 = "/TrafikInformation/AfgangAnkomst/Afgang/" + id + "/" + parts[5];
201
202	CircuitBreaker cb = CircuitBreakerManager.getManager().getCircuitBreaker("banedk");
203
204	JsoupInvocation jsoup = new JsoupInvocation(new URL("http://trafikinfo.bane.dk" + uri2), 2000);
205
206	Document doc = (Document) cb.invoke(jsoup);
207
208	Element tabel = doc.getElementById("afgangtabel");
209	if ( tabel != null) {
210	if (requiredata == true) {
211	returnVal = "";
212	} else {
213	returnVal = "<font color='red'>Data!</font>";
214	}
215	} else {
216	if (requiredata == true) {
217	returnVal = "<font color='red'>No data</font>";
218	} else {
219	returnVal = "";
220	}
221	}
222
223
224	return returnVal;
225	}
226
227
228	}