/[projects]/dao/FuldDaekningWorker/src/main/java/dk/daoas/fulddaekning/Database.java
ViewVC logotype

Annotation of /dao/FuldDaekningWorker/src/main/java/dk/daoas/fulddaekning/Database.java

Parent Directory Parent Directory | Revision Log Revision Log


Revision 2736 - (hide annotations) (download)
Fri Oct 2 09:34:36 2015 UTC (8 years, 8 months ago) by torben
File size: 9594 byte(s)
Update test mode

1 torben 2145 package dk.daoas.fulddaekning;
2    
3 torben 2152 import java.io.IOException;
4 torben 2145 import java.sql.Connection;
5 torben 2152 import java.sql.DriverManager;
6 torben 2145 import java.sql.PreparedStatement;
7     import java.sql.ResultSet;
8     import java.sql.SQLException;
9     import java.util.ArrayList;
10 torben 2247 import java.util.HashMap;
11 torben 2145 import java.util.List;
12 torben 2597 import java.util.Map;
13 torben 2152 import java.util.Properties;
14 torben 2711 import java.util.Queue;
15 torben 2328 import java.util.Set;
16     import java.util.TreeSet;
17 torben 2711 import java.util.concurrent.ConcurrentLinkedQueue;
18 torben 2145 import java.util.logging.Logger;
19    
20    
21 torben 2219
22 torben 2145 public class Database {
23     Logger logger = Logger.getLogger(Database.class.getName());
24 torben 2151
25 torben 2219 int batchCount = 0;
26 torben 2151
27 torben 2145 Connection conn;
28     PreparedStatement saveStmt;
29 torben 2247
30 torben 2707 List<Adresse> alleAdresser;
31 torben 2328 Adresse alleIkkeDaekkede[];
32 torben 2263
33 torben 2707 Map<Short,List<Adresse>> daekkedeAdresserHO = new HashMap<Short,List<Adresse>>();
34 torben 2328
35 torben 2597
36    
37    
38    
39    
40 torben 2327 DeduplicateHelper<String> husnrbogstavCache = new DeduplicateHelper<String>();
41     DeduplicateHelper<String> ruteCache = new DeduplicateHelper<String>();
42    
43 torben 2575 Set<Short> postnumre = new TreeSet<Short>();
44 torben 2328
45 torben 2585 //Map<Short, List<Adresse>> ikkeDaekkedePrPost = new HashMap<Short, List<Adresse>>();
46 torben 2328
47    
48 torben 2575 private HashMap<Short,BoundingBox> bbCache = new HashMap<Short,BoundingBox>();
49 torben 2145
50 torben 2157 public Database(SafeProperties conf) throws SQLException,IOException {
51 torben 2152 this.conn = getConnection( conf );
52 torben 2151
53 torben 2642 String sql = "INSERT INTO fulddaekning.afstand_anden_rute_ny (orgId,orgPostnr, orgAdresse,orgGadeid,orgHusnr,orgHusnrBogstav,orgRute,id,postnr,adresse,gadeid,husnr,husnrbogstav,rute,afstand,`timestamp`) "+
54     "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )";
55 torben 2145
56 torben 2150 saveStmt = conn.prepareStatement(sql);
57 torben 2151
58 torben 2145 }
59 torben 2151
60     public void resetResultTable() throws SQLException {
61 torben 2192 logger.info("Dropping old result table (if exists)");
62     String sql = "DROP TABLE IF EXISTS fulddaekning.afstand_anden_rute_ny";
63 torben 2151 conn.createStatement().executeUpdate(sql);
64 torben 2192
65     logger.info("Create new result table");
66     sql = "CREATE TABLE fulddaekning.afstand_anden_rute_ny LIKE fulddaekning.afstand_anden_rute";
67     conn.createStatement().executeUpdate(sql);
68 torben 2151 }
69 torben 2192
70     public void renameResultTables() throws SQLException {
71 torben 2223 Constants consts = Constants.getInstance();
72     String ext = consts.getTableExtension();
73    
74 torben 2192 logger.info("Dropping old backup table (if exists)");
75 torben 2223 String sql = "DROP TABLE IF EXISTS fulddaekning.afstand_anden_rute_old" + ext;
76 torben 2192 conn.createStatement().executeUpdate(sql);
77    
78     logger.info("Rename tables");
79 torben 2590 String sql2 = "RENAME TABLE fulddaekning.afstand_anden_rute" + ext + " TO fulddaekning.afstand_anden_rute_old" + ext + ", fulddaekning.afstand_anden_rute_ny TO fulddaekning.afstand_anden_rute" + ext;
80 torben 2230
81 torben 2590 logger.info("Executing: " + sql2);
82     conn.createStatement().executeUpdate(sql2);
83 torben 2247 }
84    
85 torben 2575 public BoundingBox getBoundingbox(short postnr) {
86 torben 2247 BoundingBox bb = bbCache.get(postnr);
87     return bb.clone();//never return the original / cached object
88     }
89    
90    
91 torben 2575 public Set<Short> hentPostnumreCache() {
92 torben 2328 return postnumre;
93     }
94 torben 2151
95 torben 2328
96 torben 2711 public Queue<Adresse> hentAlleIkkedaekkedeAdresser(int minPostnr, int maxPostnr) throws SQLException {
97 torben 2238
98 torben 2328 logger.info("Henter alle IKKE-daekkede adresser");
99 torben 2151
100 torben 2231 String sql = "SELECT id,a.postnr,adresse,gadeid,husnr,husnrbogstav,latitude,longitude,rute,p.distributor as ho " +
101     "FROM fulddaekning.adressetabel a " +
102     "LEFT JOIN bogleveringer.postnummerdistributor p on (a.postnr=p.postnr) " +
103 torben 2145 "WHERE rute IS NULL " + //Ingen dækning
104 torben 2716 "AND a.postnr BETWEEN ? AND ? " +
105 torben 2145 "AND latitude IS NOT NULL " +
106     "AND longitude IS NOT NULL " +
107 torben 2235 "AND gadeid IS NOT NULL " +
108 torben 2716 "AND (a.distributor IS NULL OR a.distributor<>'LUKKET') "
109     //"LIMIT 2500"
110     ;
111 torben 2736
112     if (LookupMain.test_mode == true) {
113     sql = sql + " LIMIT 100 ";
114     }
115    
116 torben 2145 PreparedStatement stmt = conn.prepareStatement(sql);
117 torben 2716 stmt.setInt(1, minPostnr);
118     stmt.setInt(2, maxPostnr);
119 torben 2151
120 torben 2328 List<Adresse> list = hentAdresseListe( stmt );
121     alleIkkeDaekkede = list.toArray( new Adresse[ list.size() ] );
122 torben 2203
123 torben 2328 logger.info("Analyserer ikke-daekkede adresser");
124 torben 2238
125 torben 2328 for (Adresse a : alleIkkeDaekkede) {
126    
127    
128     BoundingBox bbox;
129    
130     if (! postnumre.contains(a.postnr )) {
131     postnumre.add( a.postnr );
132    
133 torben 2585 bbox = new BoundingBox();
134 torben 2328
135     bbCache.put( a.postnr, bbox);
136    
137     } else {
138     bbox = bbCache.get( a.postnr);
139     }
140    
141     bbox.latitudeMax = Math.max(bbox.latitudeMax, a.latitude);
142     bbox.latitudeMin = Math.min(bbox.latitudeMin, a.latitude);
143     bbox.longitudeMax = Math.max(bbox.longitudeMax, a.longitude);
144     bbox.longitudeMin = Math.min(bbox.longitudeMin, a.longitude);
145    
146    
147 torben 2145 }
148 torben 2711 return new ConcurrentLinkedQueue<Adresse>( list );
149 torben 2145 }
150 torben 2328
151 torben 2424
152 torben 2225
153 torben 2263
154 torben 2707 public Map<Short, List<Adresse>> getDaekkedeAdresserHO() {
155 torben 2597 return daekkedeAdresserHO;
156     }
157 torben 2424
158 torben 2597 public int hentAlleDaekkedeAdresser() throws SQLException {
159 torben 2263 if ( alleAdresser == null ) {
160     String sql = "SELECT id,a.postnr,adresse,gadeid,husnr,husnrbogstav,latitude,longitude,rute,p.distributor as ho " +
161     "FROM fulddaekning.adressetabel a " +
162     "LEFT JOIN bogleveringer.postnummerdistributor p on (a.postnr=p.postnr) " +
163     "WHERE rute IS NOT NULL " +
164     "AND latitude IS NOT NULL " +
165     "AND longitude IS NOT NULL " +
166     "AND a.distributor = ? ";
167    
168     // Forward only + concur_read_only + fetchsize tvinger driver til at hente en række af gangen (bedre performance ved store result sets)
169     // Se http://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html
170     PreparedStatement stmt = conn.prepareStatement(sql, java.sql.ResultSet.TYPE_FORWARD_ONLY, java.sql.ResultSet.CONCUR_READ_ONLY);
171     stmt.setFetchSize(Integer.MIN_VALUE);
172    
173     stmt.setString(1, LookupMain.distributor);
174    
175     List<Adresse> list = hentAdresseListe( stmt );
176 torben 2585
177 torben 2707 alleAdresser = new ArrayList<Adresse>();
178 torben 2585 alleAdresser.addAll(list);
179 torben 2597
180     for(Adresse addr : list) {
181     short ho = addr.ho;
182 torben 2707 List<Adresse> hoListe = daekkedeAdresserHO.get(ho);
183 torben 2597 if ( hoListe == null) {
184 torben 2707 hoListe = new ArrayList<Adresse>();
185 torben 2597 daekkedeAdresserHO.put(ho, hoListe);
186     }
187    
188     hoListe.add(addr);
189    
190     }
191 torben 2263 }
192 torben 2597
193     return alleAdresser.size();
194 torben 2145 }
195 torben 2225
196    
197 torben 2145
198     public synchronized void gemResultat(Adresse orgAdresse, Adresse bedsteAdresse, double bedsteAfstand) throws SQLException {
199     /*String sql = "INSERT INTO fulddaekning.afstand_anden_rute_thn (orgId,orgPostnr, orgAdresse,orgGadeid,orgHusnr,orgHusnrBogstav,orgLatitude,orgLongitude,orgRute,id,postnr,adresse,gadeid,husnr,husnrbogstav,latitude,longitude,rute,afstand,`timestamp`) "+
200     "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )";
201 torben 2151
202 torben 2145 PreparedStatement saveStmt = conn.prepareStatement(sql);*/
203 torben 2151
204 torben 2145 saveStmt.setInt(1, orgAdresse.id);
205 torben 2575 saveStmt.setShort(2, orgAdresse.postnr);
206 torben 2145 saveStmt.setString(3, orgAdresse.adresse);
207     saveStmt.setInt(4, orgAdresse.gadeid);
208 torben 2575 saveStmt.setShort(5, orgAdresse.husnr);
209 torben 2145 saveStmt.setString(6, orgAdresse.husnrbogstav);
210 torben 2642 saveStmt.setString(7, orgAdresse.rute);
211 torben 2151
212    
213 torben 2642 saveStmt.setInt(8, bedsteAdresse.id);
214     saveStmt.setShort(9, bedsteAdresse.postnr);
215     saveStmt.setString(10, bedsteAdresse.adresse);
216     saveStmt.setInt(11, bedsteAdresse.gadeid);
217     saveStmt.setShort(12, bedsteAdresse.husnr);
218     saveStmt.setString(13, bedsteAdresse.husnrbogstav);
219     saveStmt.setString(14, bedsteAdresse.rute);
220 torben 2151
221 torben 2642 saveStmt.setDouble(15, bedsteAfstand);
222 torben 2151
223 torben 2219 saveStmt.addBatch();
224     batchCount++;
225 torben 2423 if (batchCount >= 1000) {
226 torben 2219 saveStmt.executeBatch();
227     batchCount = 0;
228     }
229     //saveStmt.executeUpdate();
230     //saveStmt.clearParameters();
231 torben 2151
232 torben 2145 //saveStmt.close();
233     }
234 torben 2219
235     public synchronized void saveBatch() throws SQLException{
236     saveStmt.executeBatch();
237     batchCount = 0;
238     }
239 torben 2151
240    
241    
242 torben 2145 protected ArrayList<Adresse> hentAdresseListe(PreparedStatement stmt) throws SQLException{
243 torben 2210 ArrayList<Adresse> list = new ArrayList<Adresse>( 1000000 );
244 torben 2151
245 torben 2145 //logger.info("Starting query");
246     ResultSet res = stmt.executeQuery();
247     //logger.info("Starting exec query done");
248 torben 2151
249 torben 2145 while (res.next()) {
250 torben 2585
251     double latitude = res.getDouble(7);
252     double longitude = res.getDouble(8);
253 torben 2261
254 torben 2585
255     Adresse adr = new Adresse(latitude,longitude);
256    
257 torben 2261 adr.id = res.getInt(1);
258 torben 2575 adr.postnr = res.getShort(2);
259 torben 2261 adr.adresse = res.getString(3);
260     adr.gadeid = res.getInt(4);
261 torben 2575 adr.husnr = res.getShort(5);
262 torben 2327 adr.husnrbogstav = husnrbogstavCache.getInstance( res.getString(6) );
263 torben 2585 //adr.latitude = res.getDouble(7);
264     //adr.longitude = res.getDouble(8);
265 torben 2327 adr.rute = ruteCache.getInstance( res.getString(9) );
266 torben 2575 adr.ho = res.getShort(10);
267 torben 2151
268 torben 2145 list.add(adr);
269 torben 2151
270 torben 2145 //logger.info( "Adress:" + adr);
271     }
272 torben 2151
273 torben 2145 res.close();
274     stmt.close();
275 torben 2151
276 torben 2145 return list;
277     }
278    
279 torben 2157 public Connection getConnection(SafeProperties conf) throws SQLException, IOException {
280 torben 2152
281 torben 2157 String db_host = conf.getSafeProperty("DB_HOST");
282     String db_user = conf.getSafeProperty("DB_USER");
283     String db_pass = conf.getSafeProperty("DB_PASS");
284 torben 2145
285 torben 2152
286    
287 torben 2151
288 torben 2152 Connection conn = null;
289     Properties connectionProps = new Properties();
290     connectionProps.put("user", db_user);
291     connectionProps.put("password", db_pass);
292    
293 torben 2220 //For debug output, tilføj denne til JDBC url'en: &profileSQL=true
294 torben 2152 conn = DriverManager.getConnection(
295     "jdbc:mysql://" +
296     db_host +
297 torben 2218 ":3306/?rewriteBatchedStatements=true",
298 torben 2152 connectionProps);
299     logger.info("Connected to database");
300     return conn;
301     }
302    
303 torben 2585
304 torben 2145 }

  ViewVC Help
Powered by ViewVC 1.1.20