traininfoservice/banedk/CompareStations.java

package dk.thoerup.traininfoservice.banedk;

import java.io.IOException;
import java.io.PrintWriter;
import java.net.URL;
import java.net.URLDecoder;
import java.net.URLEncoder;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;

import javax.servlet.ServletException;
import javax.servlet.annotation.WebServlet;
import javax.servlet.http.HttpServlet;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import dk.thoerup.android.traininfo.common.StationBean;
import dk.thoerup.android.traininfo.common.StationEntry;
import dk.thoerup.circuitbreaker.CircuitBreaker;
import dk.thoerup.circuitbreaker.CircuitBreakerManager;
import dk.thoerup.traininfoservice.db.StationDAO;

import dk.thoerup.genericjavautils.TimeoutMap;


@WebServlet(urlPatterns={"/CompareStations"})
public class CompareStations extends HttpServlet {
        private static final long serialVersionUID = 1L;

        Map<String,String> cache = new TimeoutMap<String,String>(600 * 1000);
       
        protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
                try {
                        String data = "";
                        String uri = request.getParameter("uri");

                        if (uri != null) {
                                boolean require = Boolean.parseBoolean( request.getParameter("requiredata") );
                                data = hasData(uri, require);
                        } else {
                                data = getData(request);
                        }

                        PrintWriter out = response.getWriter();
                        out.print(data);

                } catch (Exception e) {
                        throw new ServletException(e);
                }
        }

        String getData(HttpServletRequest request) throws Exception {
                CircuitBreaker cb = CircuitBreakerManager.getManager().getCircuitBreaker("banedk");
                
                JsoupInvocation jsoup = new JsoupInvocation(new URL("http://trafikinfo.bane.dk/Trafikinformation/Stationsliste"), 5000);
                
                
                Map<String,String> idToStation = new TreeMap<String,String>();
                Map<String,String> stationToId = new TreeMap<String,String>();
                Map<String,String> banedkStations = new TreeMap<String,String>();
                
                Set<String> dbStations = new TreeSet<String>();
        
                StringBuilder sb = new StringBuilder(); 
                sb.append("<html>");
                sb.append("<head>");
                sb.append("<script type='text/javascript' src='https://ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js'></script>\n");
                sb.append("<script type='text/javascript' src='compareloader.js'></script>\n");

                sb.append("</head>");
                sb.append("<body>");
        
                StationDAO dao = new StationDAO();
                
                try {
                        Document doc = (Document) cb.invoke(jsoup);
                        
                        Elements tables = doc.getElementsByClass("Oversigt");
                        
                        for(Element e : tables) {
                                if (e.tagName().equals("table") ){
                                        Elements links = e.getElementsByTag("a");
                                        for (Element link : links) {            
                                                String name = link.text();
                                                String href = link.attr("href");
                                                
                                                String hrefParts[] = href.split("/");
                                                String hrefCode = URLDecoder.decode(hrefParts[4], "UTF-8");
                                                
                                                idToStation.put(hrefCode, name);
                                                stationToId.put(name, hrefCode);
                                                banedkStations.put( name, href );
                                        }
                                }
                        }
                        
                        StationBean bean = dao.dumpAll();
                        
                        ////////////////////////////////
                        sb.append("<h2>Duplicate ID's in DB</h2>\n");
                        sb.append("<ul>");                      
                        Map<String,String> stationIDS = new TreeMap<String,String>();
                        for (StationEntry station : bean.entries) {
                                if (station.isRegional()) {
                                        String id = station.getRegional();
                                        String oldstation = stationIDS.get(id);
                                        if (oldstation == null) {
                                                stationIDS.put(id, station.getName());
                                        } else {
                                                sb.append("<li> Duplicate found on key=" + id + " old station=" + oldstation + " current=" + station.getName());
                                        }
                                }
                                if (station.isStrain()) {
                                        String id = station.getStrain();
                                        String oldstation= stationIDS.get(id);
                                        if (oldstation == null) {
                                                stationIDS.put(id, station.getName());
                                        } else {
                                                sb.append("<li> Duplicate found on key=" + id + " old station=" + oldstation + " current=" + station.getName());
                                        }
                                }
                        }                       
                        sb.append("</ul>");
                        
                        ////////////////////////////////                        

                        sb.append("<h2>Active stations in DB</h2>\n");
                        sb.append("<ul>");

                        
                        for (StationEntry station : bean.entries) {                             
                                if (station.isRegional() || station.isStrain()) {
                                        String hrefCode = stationToId.get(station.getName());
                                        if (hrefCode == null)
                                                hrefCode = "null";
                                        
                                        String id = "";
                                        String uri = "";
                                        if (station.isRegional() ) {            
                                                id = station.getRegional();
                                                uri = "/Trafikinformation/AfgangAnkomst/Afgang/" + id + "/Fjerntog";
                                        }
                                        if (station.isStrain() ) {              
                                                id = station.getStrain();
                                                uri = "/Trafikinformation/AfgangAnkomst/Afgang/" + id + "/S-Tog";
                                        }
                                        
                                        String idMismatch = "";
                                        if (!hrefCode.equals(id) ) {
                                                idMismatch = " ID Mismatch (DB="+id+ ", BaneDK=" + hrefCode + ")";
                                        }
                                                
                                        
                                        sb.append( "<li><a target='_blank' href='http://trafikinfo.bane.dk" + uri + "'>" + station.getName() + "</a> <span class='uri' requiredata='true' uri='" + uri + "'></span> " + idMismatch + "</li>\n" );

                                        dbStations.add(station.getName());                                      
                                }
                                

                        }
                        sb.append("</ul>\n");
                        
                } catch (Exception e) {
                        throw new ServletException(e);
                }
                
                
                sb.append("<h2>stations on website not in db</h2>\n");
                sb.append("<ul>");
                Set<String> tmpDbStations = new TreeSet<String>( dbStations );
                Set<String> tmpBanedkStations = new TreeSet<String>( banedkStations.keySet()) ;
                tmpBanedkStations.removeAll(tmpDbStations);

                
                for(String s : tmpBanedkStations) {
                        String uri = banedkStations.get(s);
                        String disabled = "";
                        //String data = hasData(uri, false);
                        
                        try {
                                boolean tmpdisabled = dao.hasDisabledStation(s);
                                if (tmpdisabled == true)
                                        disabled = " - disabled";
                        } catch (Exception e) {
                                throw new ServletException(e);
                        }
                         
                        sb.append( "<li><a target='_blank' href='http://trafikinfo.bane.dk" + uri + "'>" + s + "</a>&nbsp;<span class='uri' requiredata='false' uri='" + uri + "'></span>" + disabled + "</li>\n" );
                }                               
                sb.append("</ul>");

                
                sb.append("<h2>stations in db not on website </h2>");
                sb.append("<ul>");
                tmpDbStations = new TreeSet<String>( dbStations );
                tmpBanedkStations = new TreeSet<String>( banedkStations.keySet() ) ;
                tmpDbStations.removeAll(tmpBanedkStations);
                for(String s : tmpDbStations) {
                        sb.append( "<li>" + s + "</li>\n" );
                }                               
                sb.append("</ul>");
                sb.append("<font size='1' id='antal'></font>");
                sb.append("</body></html>\n");
                
                return sb.toString();   
                
        }
        
        String hasData(String uri, boolean requiredata) {
                String key = uri + requiredata;

                String data = cache.get(key);
                if (data == null) {
                
                        data = "<font color='yellow'>unknown</font>";
                        try {
                                data = hasDataWorker(uri,requiredata);
                                cache.put(key,data);
                        } catch (Exception e) {
                                System.out.println( e.getMessage() );
                        }
                }
                return data;
        }

        String hasDataWorker(String uri, boolean requiredata) throws Exception {
                String returnVal = ""; 
                
                String parts[] = uri.split("\\/");
                String id = URLEncoder.encode(parts[4], "ISO-8859-1");
                String uri2 = "/TrafikInformation/AfgangAnkomst/Afgang/" + id + "/" + parts[5];

                CircuitBreaker cb = CircuitBreakerManager.getManager().getCircuitBreaker("banedk");
                        
                JsoupInvocation jsoup = new JsoupInvocation(new URL("http://trafikinfo.bane.dk" + uri2), 2000);
                        
                Document doc = (Document) cb.invoke(jsoup);
                        
                Element tabel = doc.getElementById("afgangtabel");
                if ( tabel != null) {
                        if (requiredata == true) {
                                returnVal = "";
                        } else {        
                                returnVal = "<font color='red'>Data!</font>"; 
                        }
                } else {
                        if (requiredata == true) {
                                returnVal = "<font color='red'>No data</font>"; 
                        } else {
                                returnVal =  "";
                        }
                }
                        
                
                return returnVal;
        }


}
1	torben	1449	package dk.thoerup.traininfoservice.banedk;
2
3			import java.io.IOException;
4			import java.io.PrintWriter;
5			import java.net.URL;
6	torben	1906	import java.net.URLDecoder;
7	torben	1698	import java.net.URLEncoder;
8	torben	1451	import java.util.Map;
9	torben	1450	import java.util.Set;
10	torben	1451	import java.util.TreeMap;
11	torben	1450	import java.util.TreeSet;
12	torben	1449
13			import javax.servlet.ServletException;
14			import javax.servlet.annotation.WebServlet;
15			import javax.servlet.http.HttpServlet;
16			import javax.servlet.http.HttpServletRequest;
17			import javax.servlet.http.HttpServletResponse;
18
19			import org.jsoup.nodes.Document;
20			import org.jsoup.nodes.Element;
21			import org.jsoup.select.Elements;
22
23			import dk.thoerup.android.traininfo.common.StationBean;
24			import dk.thoerup.android.traininfo.common.StationEntry;
25			import dk.thoerup.circuitbreaker.CircuitBreaker;
26			import dk.thoerup.circuitbreaker.CircuitBreakerManager;
27			import dk.thoerup.traininfoservice.db.StationDAO;
28
29	torben	1698	import dk.thoerup.genericjavautils.TimeoutMap;
30	torben	1449
31	torben	1698
32	torben	1449	@WebServlet(urlPatterns={"/CompareStations"})
33			public class CompareStations extends HttpServlet {
34			private static final long serialVersionUID = 1L;
35	torben	1698
36	torben	1699	Map<String,String> cache = new TimeoutMap<String,String>(600 * 1000);
37	torben	1449
38	torben	1698	protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
39			try {
40	torben	1700	String data = "";
41			String uri = request.getParameter("uri");
42	torben	1449
43	torben	1700	if (uri != null) {
44			boolean require = Boolean.parseBoolean( request.getParameter("requiredata") );
45			data = hasData(uri, require);
46			} else {
47			data = getData(request);
48			}
49
50	torben	1698	PrintWriter out = response.getWriter();
51			out.print(data);
52
53			} catch (Exception e) {
54			throw new ServletException(e);
55			}
56			}
57
58			String getData(HttpServletRequest request) throws Exception {
59	torben	1449	CircuitBreaker cb = CircuitBreakerManager.getManager().getCircuitBreaker("banedk");
60
61			JsoupInvocation jsoup = new JsoupInvocation(new URL("http://trafikinfo.bane.dk/Trafikinformation/Stationsliste"), 5000);
62
63	torben	1908
64			Map<String,String> idToStation = new TreeMap<String,String>();
65			Map<String,String> stationToId = new TreeMap<String,String>();
66	torben	1451	Map<String,String> banedkStations = new TreeMap<String,String>();
67	torben	1908
68	torben	1450	Set<String> dbStations = new TreeSet<String>();
69	torben	1698
70			StringBuilder sb = new StringBuilder();
71			sb.append("<html>");
72	torben	1700	sb.append("<head>");
73			sb.append("<script type='text/javascript' src='https://ajax.googleapis.com/ajax/libs/jquery/1.7.1/jquery.min.js'></script>\n");
74			sb.append("<script type='text/javascript' src='compareloader.js'></script>\n");
75
76			sb.append("</head>");
77	torben	1698	sb.append("<body>");
78
79	torben	1692	StationDAO dao = new StationDAO();
80	torben	1451
81	torben	1449	try {
82			Document doc = (Document) cb.invoke(jsoup);
83
84			Elements tables = doc.getElementsByClass("Oversigt");
85
86			for(Element e : tables) {
87			if (e.tagName().equals("table") ){
88			Elements links = e.getElementsByTag("a");
89	torben	1908	for (Element link : links) {
90			String name = link.text();
91			String href = link.attr("href");
92
93			String hrefParts[] = href.split("/");
94			String hrefCode = URLDecoder.decode(hrefParts[4], "UTF-8");
95
96			idToStation.put(hrefCode, name);
97			stationToId.put(name, hrefCode);
98			banedkStations.put( name, href );
99	torben	1449	}
100			}
101			}
102	torben	1907
103			StationBean bean = dao.dumpAll();
104
105			////////////////////////////////
106			sb.append("<h2>Duplicate ID's in DB</h2>\n");
107			sb.append("<ul>");
108			Map<String,String> stationIDS = new TreeMap<String,String>();
109			for (StationEntry station : bean.entries) {
110			if (station.isRegional()) {
111			String id = station.getRegional();
112			String oldstation = stationIDS.get(id);
113			if (oldstation == null) {
114			stationIDS.put(id, station.getName());
115			} else {
116			sb.append("<li> Duplicate found on key=" + id + " old station=" + oldstation + " current=" + station.getName());
117			}
118			}
119			if (station.isStrain()) {
120			String id = station.getStrain();
121			String oldstation= stationIDS.get(id);
122			if (oldstation == null) {
123			stationIDS.put(id, station.getName());
124			} else {
125			sb.append("<li> Duplicate found on key=" + id + " old station=" + oldstation + " current=" + station.getName());
126			}
127			}
128			}
129			sb.append("</ul>");
130
131			////////////////////////////////
132	torben	1449
133	torben	1698	sb.append("<h2>Active stations in DB</h2>\n");
134			sb.append("<ul>");
135
136	torben	1907
137	torben	1906	for (StationEntry station : bean.entries) {
138	torben	1449	if (station.isRegional() \|\| station.isStrain()) {
139	torben	1908	String hrefCode = stationToId.get(station.getName());
140			if (hrefCode == null)
141			hrefCode = "null";
142	torben	1906
143			String id = "";
144			String uri = "";
145	torben	1698	if (station.isRegional() ) {
146	torben	1906	id = station.getRegional();
147			uri = "/Trafikinformation/AfgangAnkomst/Afgang/" + id + "/Fjerntog";
148	torben	1698	}
149			if (station.isStrain() ) {
150	torben	1906	id = station.getStrain();
151			uri = "/Trafikinformation/AfgangAnkomst/Afgang/" + id + "/S-Tog";
152			}
153	torben	1698
154	torben	1906	String idMismatch = "";
155			if (!hrefCode.equals(id) ) {
156			idMismatch = " ID Mismatch (DB="+id+ ", BaneDK=" + hrefCode + ")";
157	torben	1698	}
158	torben	1906
159
160			sb.append( "<li><a target='_blank' href='http://trafikinfo.bane.dk" + uri + "'>" + station.getName() + "</a> <span class='uri' requiredata='true' uri='" + uri + "'></span> " + idMismatch + "</li>\n" );
161	torben	1698
162	torben	1906	dbStations.add(station.getName());
163	torben	1449	}
164	torben	1906
165
166	torben	1449	}
167	torben	1698	sb.append("</ul>\n");
168	torben	1449
169			} catch (Exception e) {
170			throw new ServletException(e);
171			}
172
173
174
175	torben	1698	sb.append("<h2>stations on website not in db</h2>\n");
176	torben	1449	sb.append("<ul>");
177	torben	1450	Set<String> tmpDbStations = new TreeSet<String>( dbStations );
178	torben	1451	Set<String> tmpBanedkStations = new TreeSet<String>( banedkStations.keySet()) ;
179	torben	1449	tmpBanedkStations.removeAll(tmpDbStations);
180	torben	1692
181
182
183	torben	1449	for(String s : tmpBanedkStations) {
184	torben	1451	String uri = banedkStations.get(s);
185	torben	1692	String disabled = "";
186	torben	1700	//String data = hasData(uri, false);
187	torben	1693
188	torben	1692	try {
189			boolean tmpdisabled = dao.hasDisabledStation(s);
190			if (tmpdisabled == true)
191	torben	1699	disabled = " - disabled";
192	torben	1692	} catch (Exception e) {
193			throw new ServletException(e);
194			}
195
196	torben	1700	sb.append( "<li><a target='_blank' href='http://trafikinfo.bane.dk" + uri + "'>" + s + "</a> <span class='uri' requiredata='false' uri='" + uri + "'></span>" + disabled + "</li>\n" );
197	torben	1449	}
198			sb.append("</ul>");
199
200
201			sb.append("<h2>stations in db not on website </h2>");
202			sb.append("<ul>");
203	torben	1450	tmpDbStations = new TreeSet<String>( dbStations );
204	torben	1451	tmpBanedkStations = new TreeSet<String>( banedkStations.keySet() ) ;
205	torben	1449	tmpDbStations.removeAll(tmpBanedkStations);
206			for(String s : tmpDbStations) {
207	torben	1698	sb.append( "<li>" + s + "</li>\n" );
208	torben	1449	}
209			sb.append("</ul>");
210	torben	1702	sb.append("<font size='1' id='antal'></font>");
211	torben	1698	sb.append("</body></html>\n");
212	torben	1449
213	torben	1698	return sb.toString();
214	torben	1449
215	torben	1693	}
216
217	torben	1698	String hasData(String uri, boolean requiredata) {
218			String key = uri + requiredata;
219
220			String data = cache.get(key);
221			if (data == null) {
222	torben	1449
223	torben	1698	data = "<font color='yellow'>unknown</font>";
224			try {
225			data = hasDataWorker(uri,requiredata);
226			cache.put(key,data);
227			} catch (Exception e) {
228			System.out.println( e.getMessage() );
229			}
230			}
231			return data;
232			}
233
234			String hasDataWorker(String uri, boolean requiredata) throws Exception {
235			String returnVal = "";
236
237	torben	1700	String parts[] = uri.split("\\/");
238			String id = URLEncoder.encode(parts[4], "ISO-8859-1");
239			String uri2 = "/TrafikInformation/AfgangAnkomst/Afgang/" + id + "/" + parts[5];
240
241	torben	1698	CircuitBreaker cb = CircuitBreakerManager.getManager().getCircuitBreaker("banedk");
242	torben	1693
243	torben	1700	JsoupInvocation jsoup = new JsoupInvocation(new URL("http://trafikinfo.bane.dk" + uri2), 2000);
244	torben	1693
245	torben	1698	Document doc = (Document) cb.invoke(jsoup);
246	torben	1693
247	torben	1698	Element tabel = doc.getElementById("afgangtabel");
248			if ( tabel != null) {
249			if (requiredata == true) {
250			returnVal = "";
251			} else {
252	torben	1693	returnVal = "<font color='red'>Data!</font>";
253	torben	1698	}
254			} else {
255			if (requiredata == true) {
256			returnVal = "<font color='red'>No data</font>";
257	torben	1693	} else {
258			returnVal = "";
259			}
260	torben	1698	}
261	torben	1693
262	torben	1449
263	torben	1693	return returnVal;
264	torben	1449	}
265
266	torben	1692
267	torben	1449	}