/[projects]/dao/FuldDaekningWorker/src/main/java/dk/daoas/fulddaekning/Database.java
ViewVC logotype

Annotation of /dao/FuldDaekningWorker/src/main/java/dk/daoas/fulddaekning/Database.java

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2708 - (hide annotations) (download)
Sun Sep 27 18:20:41 2015 UTC (8 years, 8 months ago) by torben
Original Path: dao/FuldDaekningWorker/src/dk/daoas/fulddaekning/Database.java
File size: 9496 byte(s)
First implementation of OSRM backed calculation
1 torben 2145 package dk.daoas.fulddaekning;
2    
3 torben 2152 import java.io.IOException;
4 torben 2145 import java.sql.Connection;
5 torben 2152 import java.sql.DriverManager;
6 torben 2145 import java.sql.PreparedStatement;
7     import java.sql.ResultSet;
8     import java.sql.SQLException;
9     import java.util.ArrayList;
10 torben 2247 import java.util.HashMap;
11 torben 2145 import java.util.List;
12 torben 2597 import java.util.Map;
13 torben 2152 import java.util.Properties;
14 torben 2328 import java.util.Set;
15     import java.util.TreeSet;
16 torben 2145 import java.util.logging.Logger;
17    
18    
19 torben 2219
20 torben 2145 public class Database {
21     Logger logger = Logger.getLogger(Database.class.getName());
22 torben 2151
23 torben 2219 int batchCount = 0;
24 torben 2151
25 torben 2145 Connection conn;
26     PreparedStatement saveStmt;
27 torben 2247
28 torben 2707 List<Adresse> alleAdresser;
29 torben 2328 Adresse alleIkkeDaekkede[];
30 torben 2263
31 torben 2707 Map<Short,List<Adresse>> daekkedeAdresserHO = new HashMap<Short,List<Adresse>>();
32 torben 2328
33 torben 2597
34    
35    
36    
37    
38 torben 2327 DeduplicateHelper<String> husnrbogstavCache = new DeduplicateHelper<String>();
39     DeduplicateHelper<String> ruteCache = new DeduplicateHelper<String>();
40    
41 torben 2575 Set<Short> postnumre = new TreeSet<Short>();
42 torben 2328
43 torben 2585 //Map<Short, List<Adresse>> ikkeDaekkedePrPost = new HashMap<Short, List<Adresse>>();
44 torben 2328
45    
46 torben 2575 private HashMap<Short,BoundingBox> bbCache = new HashMap<Short,BoundingBox>();
47 torben 2145
48 torben 2157 public Database(SafeProperties conf) throws SQLException,IOException {
49 torben 2152 this.conn = getConnection( conf );
50 torben 2151
51 torben 2642 String sql = "INSERT INTO fulddaekning.afstand_anden_rute_ny (orgId,orgPostnr, orgAdresse,orgGadeid,orgHusnr,orgHusnrBogstav,orgRute,id,postnr,adresse,gadeid,husnr,husnrbogstav,rute,afstand,`timestamp`) "+
52     "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )";
53 torben 2145
54 torben 2150 saveStmt = conn.prepareStatement(sql);
55 torben 2151
56 torben 2145 }
57 torben 2151
58     public void resetResultTable() throws SQLException {
59 torben 2192 logger.info("Dropping old result table (if exists)");
60     String sql = "DROP TABLE IF EXISTS fulddaekning.afstand_anden_rute_ny";
61 torben 2151 conn.createStatement().executeUpdate(sql);
62 torben 2192
63     logger.info("Create new result table");
64     sql = "CREATE TABLE fulddaekning.afstand_anden_rute_ny LIKE fulddaekning.afstand_anden_rute";
65     conn.createStatement().executeUpdate(sql);
66 torben 2151 }
67 torben 2192
68     public void renameResultTables() throws SQLException {
69 torben 2223 Constants consts = Constants.getInstance();
70     String ext = consts.getTableExtension();
71    
72 torben 2192 logger.info("Dropping old backup table (if exists)");
73 torben 2223 String sql = "DROP TABLE IF EXISTS fulddaekning.afstand_anden_rute_old" + ext;
74 torben 2192 conn.createStatement().executeUpdate(sql);
75    
76     logger.info("Rename tables");
77 torben 2590 String sql2 = "RENAME TABLE fulddaekning.afstand_anden_rute" + ext + " TO fulddaekning.afstand_anden_rute_old" + ext + ", fulddaekning.afstand_anden_rute_ny TO fulddaekning.afstand_anden_rute" + ext;
78 torben 2230
79 torben 2590 logger.info("Executing: " + sql2);
80     conn.createStatement().executeUpdate(sql2);
81 torben 2247 }
82    
83 torben 2575 public BoundingBox getBoundingbox(short postnr) {
84 torben 2247 BoundingBox bb = bbCache.get(postnr);
85     return bb.clone();//never return the original / cached object
86     }
87    
88    
89 torben 2575 public Set<Short> hentPostnumreCache() {
90 torben 2328 return postnumre;
91     }
92 torben 2151
93 torben 2328
94 torben 2585 public Adresse[] hentAlleIkkedaekkedeAdresser(int minPostnr, int maxPostnr) throws SQLException {
95 torben 2238
96 torben 2328 logger.info("Henter alle IKKE-daekkede adresser");
97 torben 2151
98 torben 2231 String sql = "SELECT id,a.postnr,adresse,gadeid,husnr,husnrbogstav,latitude,longitude,rute,p.distributor as ho " +
99     "FROM fulddaekning.adressetabel a " +
100     "LEFT JOIN bogleveringer.postnummerdistributor p on (a.postnr=p.postnr) " +
101 torben 2145 "WHERE rute IS NULL " + //Ingen dækning
102 torben 2707 //"AND a.postnr BETWEEN ? AND ? " +
103 torben 2708 "AND a.postnr BETWEEN 8000 and 8600 " +
104     //"AND a.postnr BETWEEN 9000 AND 9300 " +
105 torben 2145 "AND latitude IS NOT NULL " +
106     "AND longitude IS NOT NULL " +
107 torben 2235 "AND gadeid IS NOT NULL " +
108 torben 2707 "AND (a.distributor IS NULL OR a.distributor<>'LUKKET') " +
109     "LIMIT 2500";
110 torben 2145 PreparedStatement stmt = conn.prepareStatement(sql);
111 torben 2707 //stmt.setInt(1, minPostnr);
112     //stmt.setInt(2, maxPostnr);
113 torben 2151
114 torben 2328 List<Adresse> list = hentAdresseListe( stmt );
115     alleIkkeDaekkede = list.toArray( new Adresse[ list.size() ] );
116 torben 2203
117 torben 2328 logger.info("Analyserer ikke-daekkede adresser");
118 torben 2238
119 torben 2328 for (Adresse a : alleIkkeDaekkede) {
120    
121    
122     BoundingBox bbox;
123    
124     if (! postnumre.contains(a.postnr )) {
125     postnumre.add( a.postnr );
126    
127 torben 2585 bbox = new BoundingBox();
128 torben 2328
129     bbCache.put( a.postnr, bbox);
130    
131     } else {
132     bbox = bbCache.get( a.postnr);
133     }
134    
135     bbox.latitudeMax = Math.max(bbox.latitudeMax, a.latitude);
136     bbox.latitudeMin = Math.min(bbox.latitudeMin, a.latitude);
137     bbox.longitudeMax = Math.max(bbox.longitudeMax, a.longitude);
138     bbox.longitudeMin = Math.min(bbox.longitudeMin, a.longitude);
139    
140    
141 torben 2145 }
142 torben 2585 return alleIkkeDaekkede;
143 torben 2145 }
144 torben 2328
145 torben 2424
146 torben 2225
147 torben 2263
148 torben 2707 public Map<Short, List<Adresse>> getDaekkedeAdresserHO() {
149 torben 2597 return daekkedeAdresserHO;
150     }
151 torben 2424
152 torben 2597 public int hentAlleDaekkedeAdresser() throws SQLException {
153 torben 2263 if ( alleAdresser == null ) {
154     String sql = "SELECT id,a.postnr,adresse,gadeid,husnr,husnrbogstav,latitude,longitude,rute,p.distributor as ho " +
155     "FROM fulddaekning.adressetabel a " +
156     "LEFT JOIN bogleveringer.postnummerdistributor p on (a.postnr=p.postnr) " +
157     "WHERE rute IS NOT NULL " +
158     "AND latitude IS NOT NULL " +
159     "AND longitude IS NOT NULL " +
160     "AND a.distributor = ? ";
161    
162     // Forward only + concur_read_only + fetchsize tvinger driver til at hente en række af gangen (bedre performance ved store result sets)
163     // Se http://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html
164     PreparedStatement stmt = conn.prepareStatement(sql, java.sql.ResultSet.TYPE_FORWARD_ONLY, java.sql.ResultSet.CONCUR_READ_ONLY);
165     stmt.setFetchSize(Integer.MIN_VALUE);
166    
167     stmt.setString(1, LookupMain.distributor);
168    
169     List<Adresse> list = hentAdresseListe( stmt );
170 torben 2585
171 torben 2707 alleAdresser = new ArrayList<Adresse>();
172 torben 2585 alleAdresser.addAll(list);
173 torben 2597
174     for(Adresse addr : list) {
175     short ho = addr.ho;
176 torben 2707 List<Adresse> hoListe = daekkedeAdresserHO.get(ho);
177 torben 2597 if ( hoListe == null) {
178 torben 2707 hoListe = new ArrayList<Adresse>();
179 torben 2597 daekkedeAdresserHO.put(ho, hoListe);
180     }
181    
182     hoListe.add(addr);
183    
184     }
185 torben 2263 }
186 torben 2597
187     return alleAdresser.size();
188 torben 2145 }
189 torben 2225
190    
191 torben 2145
192     public synchronized void gemResultat(Adresse orgAdresse, Adresse bedsteAdresse, double bedsteAfstand) throws SQLException {
193     /*String sql = "INSERT INTO fulddaekning.afstand_anden_rute_thn (orgId,orgPostnr, orgAdresse,orgGadeid,orgHusnr,orgHusnrBogstav,orgLatitude,orgLongitude,orgRute,id,postnr,adresse,gadeid,husnr,husnrbogstav,latitude,longitude,rute,afstand,`timestamp`) "+
194     "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )";
195 torben 2151
196 torben 2145 PreparedStatement saveStmt = conn.prepareStatement(sql);*/
197 torben 2151
198 torben 2145 saveStmt.setInt(1, orgAdresse.id);
199 torben 2575 saveStmt.setShort(2, orgAdresse.postnr);
200 torben 2145 saveStmt.setString(3, orgAdresse.adresse);
201     saveStmt.setInt(4, orgAdresse.gadeid);
202 torben 2575 saveStmt.setShort(5, orgAdresse.husnr);
203 torben 2145 saveStmt.setString(6, orgAdresse.husnrbogstav);
204 torben 2642 saveStmt.setString(7, orgAdresse.rute);
205 torben 2151
206    
207 torben 2642 saveStmt.setInt(8, bedsteAdresse.id);
208     saveStmt.setShort(9, bedsteAdresse.postnr);
209     saveStmt.setString(10, bedsteAdresse.adresse);
210     saveStmt.setInt(11, bedsteAdresse.gadeid);
211     saveStmt.setShort(12, bedsteAdresse.husnr);
212     saveStmt.setString(13, bedsteAdresse.husnrbogstav);
213     saveStmt.setString(14, bedsteAdresse.rute);
214 torben 2151
215 torben 2642 saveStmt.setDouble(15, bedsteAfstand);
216 torben 2151
217 torben 2219 saveStmt.addBatch();
218     batchCount++;
219 torben 2423 if (batchCount >= 1000) {
220 torben 2219 saveStmt.executeBatch();
221     batchCount = 0;
222     }
223     //saveStmt.executeUpdate();
224     //saveStmt.clearParameters();
225 torben 2151
226 torben 2145 //saveStmt.close();
227     }
228 torben 2219
229     public synchronized void saveBatch() throws SQLException{
230     saveStmt.executeBatch();
231     batchCount = 0;
232     }
233 torben 2151
234    
235    
236 torben 2145 protected ArrayList<Adresse> hentAdresseListe(PreparedStatement stmt) throws SQLException{
237 torben 2210 ArrayList<Adresse> list = new ArrayList<Adresse>( 1000000 );
238 torben 2151
239 torben 2145 //logger.info("Starting query");
240     ResultSet res = stmt.executeQuery();
241     //logger.info("Starting exec query done");
242 torben 2151
243 torben 2145 while (res.next()) {
244 torben 2585
245     double latitude = res.getDouble(7);
246     double longitude = res.getDouble(8);
247 torben 2261
248 torben 2585
249     Adresse adr = new Adresse(latitude,longitude);
250    
251 torben 2261 adr.id = res.getInt(1);
252 torben 2575 adr.postnr = res.getShort(2);
253 torben 2261 adr.adresse = res.getString(3);
254     adr.gadeid = res.getInt(4);
255 torben 2575 adr.husnr = res.getShort(5);
256 torben 2327 adr.husnrbogstav = husnrbogstavCache.getInstance( res.getString(6) );
257 torben 2585 //adr.latitude = res.getDouble(7);
258     //adr.longitude = res.getDouble(8);
259 torben 2327 adr.rute = ruteCache.getInstance( res.getString(9) );
260 torben 2575 adr.ho = res.getShort(10);
261 torben 2151
262 torben 2145 list.add(adr);
263 torben 2151
264 torben 2145 //logger.info( "Adress:" + adr);
265     }
266 torben 2151
267 torben 2145 res.close();
268     stmt.close();
269 torben 2151
270 torben 2145 return list;
271     }
272    
273 torben 2157 public Connection getConnection(SafeProperties conf) throws SQLException, IOException {
274 torben 2152
275 torben 2157 String db_host = conf.getSafeProperty("DB_HOST");
276     String db_user = conf.getSafeProperty("DB_USER");
277     String db_pass = conf.getSafeProperty("DB_PASS");
278 torben 2145
279 torben 2152
280    
281 torben 2151
282 torben 2152 Connection conn = null;
283     Properties connectionProps = new Properties();
284     connectionProps.put("user", db_user);
285     connectionProps.put("password", db_pass);
286    
287 torben 2220 //For debug output, tilføj denne til JDBC url'en: &profileSQL=true
288 torben 2152 conn = DriverManager.getConnection(
289     "jdbc:mysql://" +
290     db_host +
291 torben 2218 ":3306/?rewriteBatchedStatements=true",
292 torben 2152 connectionProps);
293     logger.info("Connected to database");
294     return conn;
295     }
296    
297 torben 2585
298 torben 2145 }

  ViewVC Help
Powered by ViewVC 1.1.20