/[projects]/dao/FuldDaekningWorker/src/main/java/dk/daoas/fulddaekning/Database.java
ViewVC logotype

Annotation of /dao/FuldDaekningWorker/src/main/java/dk/daoas/fulddaekning/Database.java

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2716 - (hide annotations) (download)
Sun Sep 27 20:14:24 2015 UTC (8 years, 8 months ago) by torben
Original Path: dao/FuldDaekningWorker/src/dk/daoas/fulddaekning/Database.java
File size: 9511 byte(s)
Back to normal operations
1 torben 2145 package dk.daoas.fulddaekning;
2    
3 torben 2152 import java.io.IOException;
4 torben 2145 import java.sql.Connection;
5 torben 2152 import java.sql.DriverManager;
6 torben 2145 import java.sql.PreparedStatement;
7     import java.sql.ResultSet;
8     import java.sql.SQLException;
9     import java.util.ArrayList;
10 torben 2247 import java.util.HashMap;
11 torben 2145 import java.util.List;
12 torben 2597 import java.util.Map;
13 torben 2152 import java.util.Properties;
14 torben 2711 import java.util.Queue;
15 torben 2328 import java.util.Set;
16     import java.util.TreeSet;
17 torben 2711 import java.util.concurrent.ConcurrentLinkedQueue;
18 torben 2145 import java.util.logging.Logger;
19    
20    
21 torben 2219
22 torben 2145 public class Database {
23     Logger logger = Logger.getLogger(Database.class.getName());
24 torben 2151
25 torben 2219 int batchCount = 0;
26 torben 2151
27 torben 2145 Connection conn;
28     PreparedStatement saveStmt;
29 torben 2247
30 torben 2707 List<Adresse> alleAdresser;
31 torben 2328 Adresse alleIkkeDaekkede[];
32 torben 2263
33 torben 2707 Map<Short,List<Adresse>> daekkedeAdresserHO = new HashMap<Short,List<Adresse>>();
34 torben 2328
35 torben 2597
36    
37    
38    
39    
40 torben 2327 DeduplicateHelper<String> husnrbogstavCache = new DeduplicateHelper<String>();
41     DeduplicateHelper<String> ruteCache = new DeduplicateHelper<String>();
42    
43 torben 2575 Set<Short> postnumre = new TreeSet<Short>();
44 torben 2328
45 torben 2585 //Map<Short, List<Adresse>> ikkeDaekkedePrPost = new HashMap<Short, List<Adresse>>();
46 torben 2328
47    
48 torben 2575 private HashMap<Short,BoundingBox> bbCache = new HashMap<Short,BoundingBox>();
49 torben 2145
50 torben 2157 public Database(SafeProperties conf) throws SQLException,IOException {
51 torben 2152 this.conn = getConnection( conf );
52 torben 2151
53 torben 2642 String sql = "INSERT INTO fulddaekning.afstand_anden_rute_ny (orgId,orgPostnr, orgAdresse,orgGadeid,orgHusnr,orgHusnrBogstav,orgRute,id,postnr,adresse,gadeid,husnr,husnrbogstav,rute,afstand,`timestamp`) "+
54     "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )";
55 torben 2145
56 torben 2150 saveStmt = conn.prepareStatement(sql);
57 torben 2151
58 torben 2145 }
59 torben 2151
60     public void resetResultTable() throws SQLException {
61 torben 2192 logger.info("Dropping old result table (if exists)");
62     String sql = "DROP TABLE IF EXISTS fulddaekning.afstand_anden_rute_ny";
63 torben 2151 conn.createStatement().executeUpdate(sql);
64 torben 2192
65     logger.info("Create new result table");
66     sql = "CREATE TABLE fulddaekning.afstand_anden_rute_ny LIKE fulddaekning.afstand_anden_rute";
67     conn.createStatement().executeUpdate(sql);
68 torben 2151 }
69 torben 2192
70     public void renameResultTables() throws SQLException {
71 torben 2223 Constants consts = Constants.getInstance();
72     String ext = consts.getTableExtension();
73    
74 torben 2192 logger.info("Dropping old backup table (if exists)");
75 torben 2223 String sql = "DROP TABLE IF EXISTS fulddaekning.afstand_anden_rute_old" + ext;
76 torben 2192 conn.createStatement().executeUpdate(sql);
77    
78     logger.info("Rename tables");
79 torben 2590 String sql2 = "RENAME TABLE fulddaekning.afstand_anden_rute" + ext + " TO fulddaekning.afstand_anden_rute_old" + ext + ", fulddaekning.afstand_anden_rute_ny TO fulddaekning.afstand_anden_rute" + ext;
80 torben 2230
81 torben 2590 logger.info("Executing: " + sql2);
82     conn.createStatement().executeUpdate(sql2);
83 torben 2247 }
84    
85 torben 2575 public BoundingBox getBoundingbox(short postnr) {
86 torben 2247 BoundingBox bb = bbCache.get(postnr);
87     return bb.clone();//never return the original / cached object
88     }
89    
90    
91 torben 2575 public Set<Short> hentPostnumreCache() {
92 torben 2328 return postnumre;
93     }
94 torben 2151
95 torben 2328
96 torben 2711 public Queue<Adresse> hentAlleIkkedaekkedeAdresser(int minPostnr, int maxPostnr) throws SQLException {
97 torben 2238
98 torben 2328 logger.info("Henter alle IKKE-daekkede adresser");
99 torben 2151
100 torben 2231 String sql = "SELECT id,a.postnr,adresse,gadeid,husnr,husnrbogstav,latitude,longitude,rute,p.distributor as ho " +
101     "FROM fulddaekning.adressetabel a " +
102     "LEFT JOIN bogleveringer.postnummerdistributor p on (a.postnr=p.postnr) " +
103 torben 2145 "WHERE rute IS NULL " + //Ingen dækning
104 torben 2716 "AND a.postnr BETWEEN ? AND ? " +
105 torben 2145 "AND latitude IS NOT NULL " +
106     "AND longitude IS NOT NULL " +
107 torben 2235 "AND gadeid IS NOT NULL " +
108 torben 2716 "AND (a.distributor IS NULL OR a.distributor<>'LUKKET') "
109     //"LIMIT 2500"
110     ;
111 torben 2145 PreparedStatement stmt = conn.prepareStatement(sql);
112 torben 2716 stmt.setInt(1, minPostnr);
113     stmt.setInt(2, maxPostnr);
114 torben 2151
115 torben 2328 List<Adresse> list = hentAdresseListe( stmt );
116     alleIkkeDaekkede = list.toArray( new Adresse[ list.size() ] );
117 torben 2203
118 torben 2328 logger.info("Analyserer ikke-daekkede adresser");
119 torben 2238
120 torben 2328 for (Adresse a : alleIkkeDaekkede) {
121    
122    
123     BoundingBox bbox;
124    
125     if (! postnumre.contains(a.postnr )) {
126     postnumre.add( a.postnr );
127    
128 torben 2585 bbox = new BoundingBox();
129 torben 2328
130     bbCache.put( a.postnr, bbox);
131    
132     } else {
133     bbox = bbCache.get( a.postnr);
134     }
135    
136     bbox.latitudeMax = Math.max(bbox.latitudeMax, a.latitude);
137     bbox.latitudeMin = Math.min(bbox.latitudeMin, a.latitude);
138     bbox.longitudeMax = Math.max(bbox.longitudeMax, a.longitude);
139     bbox.longitudeMin = Math.min(bbox.longitudeMin, a.longitude);
140    
141    
142 torben 2145 }
143 torben 2711 return new ConcurrentLinkedQueue<Adresse>( list );
144 torben 2145 }
145 torben 2328
146 torben 2424
147 torben 2225
148 torben 2263
149 torben 2707 public Map<Short, List<Adresse>> getDaekkedeAdresserHO() {
150 torben 2597 return daekkedeAdresserHO;
151     }
152 torben 2424
153 torben 2597 public int hentAlleDaekkedeAdresser() throws SQLException {
154 torben 2263 if ( alleAdresser == null ) {
155     String sql = "SELECT id,a.postnr,adresse,gadeid,husnr,husnrbogstav,latitude,longitude,rute,p.distributor as ho " +
156     "FROM fulddaekning.adressetabel a " +
157     "LEFT JOIN bogleveringer.postnummerdistributor p on (a.postnr=p.postnr) " +
158     "WHERE rute IS NOT NULL " +
159     "AND latitude IS NOT NULL " +
160     "AND longitude IS NOT NULL " +
161     "AND a.distributor = ? ";
162    
163     // Forward only + concur_read_only + fetchsize tvinger driver til at hente en række af gangen (bedre performance ved store result sets)
164     // Se http://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html
165     PreparedStatement stmt = conn.prepareStatement(sql, java.sql.ResultSet.TYPE_FORWARD_ONLY, java.sql.ResultSet.CONCUR_READ_ONLY);
166     stmt.setFetchSize(Integer.MIN_VALUE);
167    
168     stmt.setString(1, LookupMain.distributor);
169    
170     List<Adresse> list = hentAdresseListe( stmt );
171 torben 2585
172 torben 2707 alleAdresser = new ArrayList<Adresse>();
173 torben 2585 alleAdresser.addAll(list);
174 torben 2597
175     for(Adresse addr : list) {
176     short ho = addr.ho;
177 torben 2707 List<Adresse> hoListe = daekkedeAdresserHO.get(ho);
178 torben 2597 if ( hoListe == null) {
179 torben 2707 hoListe = new ArrayList<Adresse>();
180 torben 2597 daekkedeAdresserHO.put(ho, hoListe);
181     }
182    
183     hoListe.add(addr);
184    
185     }
186 torben 2263 }
187 torben 2597
188     return alleAdresser.size();
189 torben 2145 }
190 torben 2225
191    
192 torben 2145
193     public synchronized void gemResultat(Adresse orgAdresse, Adresse bedsteAdresse, double bedsteAfstand) throws SQLException {
194     /*String sql = "INSERT INTO fulddaekning.afstand_anden_rute_thn (orgId,orgPostnr, orgAdresse,orgGadeid,orgHusnr,orgHusnrBogstav,orgLatitude,orgLongitude,orgRute,id,postnr,adresse,gadeid,husnr,husnrbogstav,latitude,longitude,rute,afstand,`timestamp`) "+
195     "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )";
196 torben 2151
197 torben 2145 PreparedStatement saveStmt = conn.prepareStatement(sql);*/
198 torben 2151
199 torben 2145 saveStmt.setInt(1, orgAdresse.id);
200 torben 2575 saveStmt.setShort(2, orgAdresse.postnr);
201 torben 2145 saveStmt.setString(3, orgAdresse.adresse);
202     saveStmt.setInt(4, orgAdresse.gadeid);
203 torben 2575 saveStmt.setShort(5, orgAdresse.husnr);
204 torben 2145 saveStmt.setString(6, orgAdresse.husnrbogstav);
205 torben 2642 saveStmt.setString(7, orgAdresse.rute);
206 torben 2151
207    
208 torben 2642 saveStmt.setInt(8, bedsteAdresse.id);
209     saveStmt.setShort(9, bedsteAdresse.postnr);
210     saveStmt.setString(10, bedsteAdresse.adresse);
211     saveStmt.setInt(11, bedsteAdresse.gadeid);
212     saveStmt.setShort(12, bedsteAdresse.husnr);
213     saveStmt.setString(13, bedsteAdresse.husnrbogstav);
214     saveStmt.setString(14, bedsteAdresse.rute);
215 torben 2151
216 torben 2642 saveStmt.setDouble(15, bedsteAfstand);
217 torben 2151
218 torben 2219 saveStmt.addBatch();
219     batchCount++;
220 torben 2423 if (batchCount >= 1000) {
221 torben 2219 saveStmt.executeBatch();
222     batchCount = 0;
223     }
224     //saveStmt.executeUpdate();
225     //saveStmt.clearParameters();
226 torben 2151
227 torben 2145 //saveStmt.close();
228     }
229 torben 2219
230     public synchronized void saveBatch() throws SQLException{
231     saveStmt.executeBatch();
232     batchCount = 0;
233     }
234 torben 2151
235    
236    
237 torben 2145 protected ArrayList<Adresse> hentAdresseListe(PreparedStatement stmt) throws SQLException{
238 torben 2210 ArrayList<Adresse> list = new ArrayList<Adresse>( 1000000 );
239 torben 2151
240 torben 2145 //logger.info("Starting query");
241     ResultSet res = stmt.executeQuery();
242     //logger.info("Starting exec query done");
243 torben 2151
244 torben 2145 while (res.next()) {
245 torben 2585
246     double latitude = res.getDouble(7);
247     double longitude = res.getDouble(8);
248 torben 2261
249 torben 2585
250     Adresse adr = new Adresse(latitude,longitude);
251    
252 torben 2261 adr.id = res.getInt(1);
253 torben 2575 adr.postnr = res.getShort(2);
254 torben 2261 adr.adresse = res.getString(3);
255     adr.gadeid = res.getInt(4);
256 torben 2575 adr.husnr = res.getShort(5);
257 torben 2327 adr.husnrbogstav = husnrbogstavCache.getInstance( res.getString(6) );
258 torben 2585 //adr.latitude = res.getDouble(7);
259     //adr.longitude = res.getDouble(8);
260 torben 2327 adr.rute = ruteCache.getInstance( res.getString(9) );
261 torben 2575 adr.ho = res.getShort(10);
262 torben 2151
263 torben 2145 list.add(adr);
264 torben 2151
265 torben 2145 //logger.info( "Adress:" + adr);
266     }
267 torben 2151
268 torben 2145 res.close();
269     stmt.close();
270 torben 2151
271 torben 2145 return list;
272     }
273    
274 torben 2157 public Connection getConnection(SafeProperties conf) throws SQLException, IOException {
275 torben 2152
276 torben 2157 String db_host = conf.getSafeProperty("DB_HOST");
277     String db_user = conf.getSafeProperty("DB_USER");
278     String db_pass = conf.getSafeProperty("DB_PASS");
279 torben 2145
280 torben 2152
281    
282 torben 2151
283 torben 2152 Connection conn = null;
284     Properties connectionProps = new Properties();
285     connectionProps.put("user", db_user);
286     connectionProps.put("password", db_pass);
287    
288 torben 2220 //For debug output, tilføj denne til JDBC url'en: &profileSQL=true
289 torben 2152 conn = DriverManager.getConnection(
290     "jdbc:mysql://" +
291     db_host +
292 torben 2218 ":3306/?rewriteBatchedStatements=true",
293 torben 2152 connectionProps);
294     logger.info("Connected to database");
295     return conn;
296     }
297    
298 torben 2585
299 torben 2145 }

  ViewVC Help
Powered by ViewVC 1.1.20