/[projects]/dao/FuldDaekningWorker/src/main/java/dk/daoas/fulddaekning/Database.java
ViewVC logotype

Annotation of /dao/FuldDaekningWorker/src/main/java/dk/daoas/fulddaekning/Database.java

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2741 - (hide annotations) (download)
Tue Oct 6 21:26:55 2015 UTC (8 years, 7 months ago) by torben
File size: 9795 byte(s)
streamline osrm variants - and enhance test parameter
1 torben 2145 package dk.daoas.fulddaekning;
2    
3 torben 2152 import java.io.IOException;
4 torben 2145 import java.sql.Connection;
5 torben 2152 import java.sql.DriverManager;
6 torben 2145 import java.sql.PreparedStatement;
7     import java.sql.ResultSet;
8     import java.sql.SQLException;
9     import java.util.ArrayList;
10 torben 2247 import java.util.HashMap;
11 torben 2145 import java.util.List;
12 torben 2597 import java.util.Map;
13 torben 2152 import java.util.Properties;
14 torben 2711 import java.util.Queue;
15 torben 2328 import java.util.Set;
16     import java.util.TreeSet;
17 torben 2711 import java.util.concurrent.ConcurrentLinkedQueue;
18 torben 2145 import java.util.logging.Logger;
19    
20    
21 torben 2219
22 torben 2145 public class Database {
23     Logger logger = Logger.getLogger(Database.class.getName());
24 torben 2151
25 torben 2219 int batchCount = 0;
26 torben 2151
27 torben 2145 Connection conn;
28     PreparedStatement saveStmt;
29 torben 2247
30 torben 2707 List<Adresse> alleAdresser;
31 torben 2328 Adresse alleIkkeDaekkede[];
32 torben 2263
33 torben 2707 Map<Short,List<Adresse>> daekkedeAdresserHO = new HashMap<Short,List<Adresse>>();
34 torben 2328
35 torben 2597
36    
37    
38    
39    
40 torben 2327 DeduplicateHelper<String> husnrbogstavCache = new DeduplicateHelper<String>();
41     DeduplicateHelper<String> ruteCache = new DeduplicateHelper<String>();
42    
43 torben 2575 Set<Short> postnumre = new TreeSet<Short>();
44 torben 2328
45 torben 2585 //Map<Short, List<Adresse>> ikkeDaekkedePrPost = new HashMap<Short, List<Adresse>>();
46 torben 2328
47    
48 torben 2575 private HashMap<Short,BoundingBox> bbCache = new HashMap<Short,BoundingBox>();
49 torben 2145
50 torben 2157 public Database(SafeProperties conf) throws SQLException,IOException {
51 torben 2152 this.conn = getConnection( conf );
52 torben 2151
53 torben 2642 String sql = "INSERT INTO fulddaekning.afstand_anden_rute_ny (orgId,orgPostnr, orgAdresse,orgGadeid,orgHusnr,orgHusnrBogstav,orgRute,id,postnr,adresse,gadeid,husnr,husnrbogstav,rute,afstand,`timestamp`) "+
54     "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )";
55 torben 2145
56 torben 2150 saveStmt = conn.prepareStatement(sql);
57 torben 2151
58 torben 2145 }
59 torben 2151
60     public void resetResultTable() throws SQLException {
61 torben 2192 logger.info("Dropping old result table (if exists)");
62     String sql = "DROP TABLE IF EXISTS fulddaekning.afstand_anden_rute_ny";
63 torben 2151 conn.createStatement().executeUpdate(sql);
64 torben 2192
65     logger.info("Create new result table");
66     sql = "CREATE TABLE fulddaekning.afstand_anden_rute_ny LIKE fulddaekning.afstand_anden_rute";
67     conn.createStatement().executeUpdate(sql);
68 torben 2151 }
69 torben 2192
70     public void renameResultTables() throws SQLException {
71 torben 2741 if (LookupMain.test_mode == true) {
72     throw new RuntimeException("Can not rename tables in test mode");
73     }
74    
75 torben 2223 Constants consts = Constants.getInstance();
76     String ext = consts.getTableExtension();
77    
78 torben 2192 logger.info("Dropping old backup table (if exists)");
79 torben 2223 String sql = "DROP TABLE IF EXISTS fulddaekning.afstand_anden_rute_old" + ext;
80 torben 2192 conn.createStatement().executeUpdate(sql);
81    
82     logger.info("Rename tables");
83 torben 2590 String sql2 = "RENAME TABLE fulddaekning.afstand_anden_rute" + ext + " TO fulddaekning.afstand_anden_rute_old" + ext + ", fulddaekning.afstand_anden_rute_ny TO fulddaekning.afstand_anden_rute" + ext;
84 torben 2230
85 torben 2590 logger.info("Executing: " + sql2);
86     conn.createStatement().executeUpdate(sql2);
87 torben 2247 }
88    
89 torben 2575 public BoundingBox getBoundingbox(short postnr) {
90 torben 2247 BoundingBox bb = bbCache.get(postnr);
91     return bb.clone();//never return the original / cached object
92     }
93    
94    
95 torben 2575 public Set<Short> hentPostnumreCache() {
96 torben 2328 return postnumre;
97     }
98 torben 2151
99 torben 2328
100 torben 2711 public Queue<Adresse> hentAlleIkkedaekkedeAdresser(int minPostnr, int maxPostnr) throws SQLException {
101 torben 2238
102 torben 2328 logger.info("Henter alle IKKE-daekkede adresser");
103 torben 2151
104 torben 2231 String sql = "SELECT id,a.postnr,adresse,gadeid,husnr,husnrbogstav,latitude,longitude,rute,p.distributor as ho " +
105     "FROM fulddaekning.adressetabel a " +
106     "LEFT JOIN bogleveringer.postnummerdistributor p on (a.postnr=p.postnr) " +
107 torben 2145 "WHERE rute IS NULL " + //Ingen dækning
108 torben 2716 "AND a.postnr BETWEEN ? AND ? " +
109 torben 2145 "AND latitude IS NOT NULL " +
110     "AND longitude IS NOT NULL " +
111 torben 2235 "AND gadeid IS NOT NULL " +
112 torben 2716 "AND (a.distributor IS NULL OR a.distributor<>'LUKKET') "
113     ;
114 torben 2736
115     if (LookupMain.test_mode == true) {
116     sql = sql + " LIMIT 100 ";
117     }
118    
119 torben 2145 PreparedStatement stmt = conn.prepareStatement(sql);
120 torben 2716 stmt.setInt(1, minPostnr);
121     stmt.setInt(2, maxPostnr);
122 torben 2151
123 torben 2328 List<Adresse> list = hentAdresseListe( stmt );
124     alleIkkeDaekkede = list.toArray( new Adresse[ list.size() ] );
125 torben 2203
126 torben 2328 logger.info("Analyserer ikke-daekkede adresser");
127 torben 2238
128 torben 2328 for (Adresse a : alleIkkeDaekkede) {
129    
130    
131     BoundingBox bbox;
132    
133     if (! postnumre.contains(a.postnr )) {
134     postnumre.add( a.postnr );
135    
136 torben 2585 bbox = new BoundingBox();
137 torben 2328
138     bbCache.put( a.postnr, bbox);
139    
140     } else {
141     bbox = bbCache.get( a.postnr);
142     }
143    
144     bbox.latitudeMax = Math.max(bbox.latitudeMax, a.latitude);
145     bbox.latitudeMin = Math.min(bbox.latitudeMin, a.latitude);
146     bbox.longitudeMax = Math.max(bbox.longitudeMax, a.longitude);
147     bbox.longitudeMin = Math.min(bbox.longitudeMin, a.longitude);
148    
149    
150 torben 2145 }
151 torben 2711 return new ConcurrentLinkedQueue<Adresse>( list );
152 torben 2145 }
153 torben 2328
154 torben 2424
155 torben 2225
156 torben 2263
157 torben 2707 public Map<Short, List<Adresse>> getDaekkedeAdresserHO() {
158 torben 2597 return daekkedeAdresserHO;
159     }
160 torben 2424
161 torben 2597 public int hentAlleDaekkedeAdresser() throws SQLException {
162 torben 2263 if ( alleAdresser == null ) {
163     String sql = "SELECT id,a.postnr,adresse,gadeid,husnr,husnrbogstav,latitude,longitude,rute,p.distributor as ho " +
164     "FROM fulddaekning.adressetabel a " +
165     "LEFT JOIN bogleveringer.postnummerdistributor p on (a.postnr=p.postnr) " +
166     "WHERE rute IS NOT NULL " +
167     "AND latitude IS NOT NULL " +
168     "AND longitude IS NOT NULL " +
169     "AND a.distributor = ? ";
170 torben 2741
171     if (LookupMain.test_mode == true) {
172     sql = sql + " AND a.postnr BETWEEN 6000 and 7200 ";
173     }
174 torben 2263
175     // Forward only + concur_read_only + fetchsize tvinger driver til at hente en række af gangen (bedre performance ved store result sets)
176     // Se http://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html
177     PreparedStatement stmt = conn.prepareStatement(sql, java.sql.ResultSet.TYPE_FORWARD_ONLY, java.sql.ResultSet.CONCUR_READ_ONLY);
178     stmt.setFetchSize(Integer.MIN_VALUE);
179    
180     stmt.setString(1, LookupMain.distributor);
181    
182     List<Adresse> list = hentAdresseListe( stmt );
183 torben 2585
184 torben 2707 alleAdresser = new ArrayList<Adresse>();
185 torben 2585 alleAdresser.addAll(list);
186 torben 2597
187     for(Adresse addr : list) {
188     short ho = addr.ho;
189 torben 2707 List<Adresse> hoListe = daekkedeAdresserHO.get(ho);
190 torben 2597 if ( hoListe == null) {
191 torben 2707 hoListe = new ArrayList<Adresse>();
192 torben 2597 daekkedeAdresserHO.put(ho, hoListe);
193     }
194    
195     hoListe.add(addr);
196    
197     }
198 torben 2263 }
199 torben 2597
200     return alleAdresser.size();
201 torben 2145 }
202 torben 2225
203    
204 torben 2145
205     public synchronized void gemResultat(Adresse orgAdresse, Adresse bedsteAdresse, double bedsteAfstand) throws SQLException {
206     /*String sql = "INSERT INTO fulddaekning.afstand_anden_rute_thn (orgId,orgPostnr, orgAdresse,orgGadeid,orgHusnr,orgHusnrBogstav,orgLatitude,orgLongitude,orgRute,id,postnr,adresse,gadeid,husnr,husnrbogstav,latitude,longitude,rute,afstand,`timestamp`) "+
207     "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )";
208 torben 2151
209 torben 2145 PreparedStatement saveStmt = conn.prepareStatement(sql);*/
210 torben 2151
211 torben 2145 saveStmt.setInt(1, orgAdresse.id);
212 torben 2575 saveStmt.setShort(2, orgAdresse.postnr);
213 torben 2145 saveStmt.setString(3, orgAdresse.adresse);
214     saveStmt.setInt(4, orgAdresse.gadeid);
215 torben 2575 saveStmt.setShort(5, orgAdresse.husnr);
216 torben 2145 saveStmt.setString(6, orgAdresse.husnrbogstav);
217 torben 2642 saveStmt.setString(7, orgAdresse.rute);
218 torben 2151
219    
220 torben 2642 saveStmt.setInt(8, bedsteAdresse.id);
221     saveStmt.setShort(9, bedsteAdresse.postnr);
222     saveStmt.setString(10, bedsteAdresse.adresse);
223     saveStmt.setInt(11, bedsteAdresse.gadeid);
224     saveStmt.setShort(12, bedsteAdresse.husnr);
225     saveStmt.setString(13, bedsteAdresse.husnrbogstav);
226     saveStmt.setString(14, bedsteAdresse.rute);
227 torben 2151
228 torben 2642 saveStmt.setDouble(15, bedsteAfstand);
229 torben 2151
230 torben 2219 saveStmt.addBatch();
231     batchCount++;
232 torben 2423 if (batchCount >= 1000) {
233 torben 2219 saveStmt.executeBatch();
234     batchCount = 0;
235     }
236     //saveStmt.executeUpdate();
237     //saveStmt.clearParameters();
238 torben 2151
239 torben 2145 //saveStmt.close();
240     }
241 torben 2219
242     public synchronized void saveBatch() throws SQLException{
243     saveStmt.executeBatch();
244     batchCount = 0;
245     }
246 torben 2151
247    
248    
249 torben 2145 protected ArrayList<Adresse> hentAdresseListe(PreparedStatement stmt) throws SQLException{
250 torben 2210 ArrayList<Adresse> list = new ArrayList<Adresse>( 1000000 );
251 torben 2151
252 torben 2145 //logger.info("Starting query");
253     ResultSet res = stmt.executeQuery();
254     //logger.info("Starting exec query done");
255 torben 2151
256 torben 2145 while (res.next()) {
257 torben 2585
258     double latitude = res.getDouble(7);
259     double longitude = res.getDouble(8);
260 torben 2261
261 torben 2585
262     Adresse adr = new Adresse(latitude,longitude);
263    
264 torben 2261 adr.id = res.getInt(1);
265 torben 2575 adr.postnr = res.getShort(2);
266 torben 2261 adr.adresse = res.getString(3);
267     adr.gadeid = res.getInt(4);
268 torben 2575 adr.husnr = res.getShort(5);
269 torben 2327 adr.husnrbogstav = husnrbogstavCache.getInstance( res.getString(6) );
270 torben 2585 //adr.latitude = res.getDouble(7);
271     //adr.longitude = res.getDouble(8);
272 torben 2327 adr.rute = ruteCache.getInstance( res.getString(9) );
273 torben 2575 adr.ho = res.getShort(10);
274 torben 2151
275 torben 2145 list.add(adr);
276 torben 2151
277 torben 2145 //logger.info( "Adress:" + adr);
278     }
279 torben 2151
280 torben 2145 res.close();
281     stmt.close();
282 torben 2151
283 torben 2145 return list;
284     }
285    
286 torben 2157 public Connection getConnection(SafeProperties conf) throws SQLException, IOException {
287 torben 2152
288 torben 2157 String db_host = conf.getSafeProperty("DB_HOST");
289     String db_user = conf.getSafeProperty("DB_USER");
290     String db_pass = conf.getSafeProperty("DB_PASS");
291 torben 2145
292 torben 2152
293    
294 torben 2151
295 torben 2152 Connection conn = null;
296     Properties connectionProps = new Properties();
297     connectionProps.put("user", db_user);
298     connectionProps.put("password", db_pass);
299    
300 torben 2220 //For debug output, tilføj denne til JDBC url'en: &profileSQL=true
301 torben 2152 conn = DriverManager.getConnection(
302     "jdbc:mysql://" +
303     db_host +
304 torben 2218 ":3306/?rewriteBatchedStatements=true",
305 torben 2152 connectionProps);
306     logger.info("Connected to database");
307     return conn;
308     }
309    
310 torben 2585
311 torben 2145 }

  ViewVC Help
Powered by ViewVC 1.1.20