1 |
package dk.daoas.fulddaekning; |
package dk.daoas.fulddaekning; |
2 |
|
|
3 |
|
import java.io.IOException; |
4 |
import java.sql.Connection; |
import java.sql.Connection; |
5 |
|
import java.sql.DriverManager; |
6 |
import java.sql.PreparedStatement; |
import java.sql.PreparedStatement; |
7 |
import java.sql.ResultSet; |
import java.sql.ResultSet; |
8 |
import java.sql.SQLException; |
import java.sql.SQLException; |
9 |
import java.util.ArrayList; |
import java.util.ArrayList; |
10 |
|
import java.util.HashMap; |
11 |
import java.util.List; |
import java.util.List; |
12 |
|
import java.util.Map; |
13 |
|
import java.util.Properties; |
14 |
import java.util.Queue; |
import java.util.Queue; |
15 |
|
import java.util.Set; |
16 |
|
import java.util.TreeSet; |
17 |
import java.util.concurrent.ConcurrentLinkedQueue; |
import java.util.concurrent.ConcurrentLinkedQueue; |
18 |
import java.util.logging.Logger; |
import java.util.logging.Logger; |
19 |
|
|
20 |
|
|
21 |
|
|
22 |
public class Database { |
public class Database { |
23 |
Logger logger = Logger.getLogger(Database.class.getName()); |
Logger logger = Logger.getLogger(Database.class.getName()); |
24 |
|
|
25 |
|
int batchCount = 0; |
26 |
|
|
27 |
Connection conn; |
Connection conn; |
28 |
PreparedStatement saveStmt; |
PreparedStatement saveStmt; |
29 |
|
|
30 |
|
List<Adresse> alleAdresser; |
31 |
|
Adresse alleIkkeDaekkede[]; |
32 |
|
|
33 |
|
Map<Short,List<Adresse>> daekkedeAdresserHO = new HashMap<Short,List<Adresse>>(); |
34 |
|
|
35 |
|
|
36 |
|
|
37 |
|
|
38 |
|
|
39 |
public Database(Connection conn) throws SQLException { |
|
40 |
this.conn = conn; |
DeduplicateHelper<String> husnrbogstavCache = new DeduplicateHelper<String>(); |
41 |
|
DeduplicateHelper<String> ruteCache = new DeduplicateHelper<String>(); |
42 |
String sql = "INSERT INTO fulddaekning.afstand_anden_rute_thn (orgId,orgPostnr, orgAdresse,orgGadeid,orgHusnr,orgHusnrBogstav,orgLatitude,orgLongitude,orgRute,id,postnr,adresse,gadeid,husnr,husnrbogstav,latitude,longitude,rute,afstand,`timestamp`) "+ |
|
43 |
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )"; |
Set<Short> postnumre = new TreeSet<Short>(); |
44 |
|
|
45 |
|
//Map<Short, List<Adresse>> ikkeDaekkedePrPost = new HashMap<Short, List<Adresse>>(); |
46 |
|
|
47 |
|
|
48 |
|
private HashMap<Short,BoundingBox> bbCache = new HashMap<Short,BoundingBox>(); |
49 |
|
|
50 |
|
public Database(SafeProperties conf) throws SQLException,IOException { |
51 |
|
this.conn = getConnection( conf ); |
52 |
|
|
53 |
|
String sql = "INSERT INTO fulddaekning.afstand_anden_rute_ny (orgId,orgPostnr, orgAdresse,orgGadeid,orgHusnr,orgHusnrBogstav,orgRute,id,postnr,adresse,gadeid,husnr,husnrbogstav,rute,afstand,`timestamp`) "+ |
54 |
|
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )"; |
55 |
|
|
56 |
saveStmt = conn.prepareStatement(sql); |
saveStmt = conn.prepareStatement(sql); |
57 |
|
|
58 |
} |
} |
59 |
|
|
60 |
|
public void resetResultTable() throws SQLException { |
61 |
|
logger.info("Dropping old result table (if exists)"); |
62 |
|
String sql = "DROP TABLE IF EXISTS fulddaekning.afstand_anden_rute_ny"; |
63 |
|
conn.createStatement().executeUpdate(sql); |
64 |
|
|
65 |
|
logger.info("Create new result table"); |
66 |
|
sql = "CREATE TABLE fulddaekning.afstand_anden_rute_ny LIKE fulddaekning.afstand_anden_rute"; |
67 |
|
conn.createStatement().executeUpdate(sql); |
68 |
|
} |
69 |
|
|
70 |
public BoundingBox getBoundingbox(int postnr) throws SQLException { |
public void renameResultTables() throws SQLException { |
71 |
|
Constants consts = Constants.getInstance(); |
72 |
String sql = |
String ext = consts.getTableExtension(); |
73 |
"select max(latitude) latmax, min(latitude) latmin, max(longitude) lngmax,min(longitude) lngmin " + |
|
74 |
"from fulddaekning.adressetabel WHERE postnr=? and rute is null;"; |
logger.info("Dropping old backup table (if exists)"); |
75 |
|
String sql = "DROP TABLE IF EXISTS fulddaekning.afstand_anden_rute_old" + ext; |
76 |
PreparedStatement stmt = conn.prepareStatement(sql); |
conn.createStatement().executeUpdate(sql); |
|
stmt.setInt(1, postnr); |
|
|
|
|
|
ResultSet res = stmt.executeQuery(); |
|
|
res.next(); //query returnerer altid 1 række |
|
|
|
|
|
BoundingBox bbox = new BoundingBox(); |
|
|
bbox.latitudeMax = res.getDouble("latmax"); |
|
|
bbox.latitudeMin = res.getDouble("latmin"); |
|
|
bbox.longitudeMax = res.getDouble("lngmax"); |
|
|
bbox.longitudeMin = res.getDouble("lngmin"); |
|
|
|
|
|
res.close(); |
|
|
stmt.close(); |
|
77 |
|
|
78 |
return bbox; |
logger.info("Rename tables"); |
79 |
|
String sql2 = "RENAME TABLE fulddaekning.afstand_anden_rute" + ext + " TO fulddaekning.afstand_anden_rute_old" + ext + ", fulddaekning.afstand_anden_rute_ny TO fulddaekning.afstand_anden_rute" + ext; |
80 |
|
|
81 |
|
logger.info("Executing: " + sql2); |
82 |
|
conn.createStatement().executeUpdate(sql2); |
83 |
|
} |
84 |
|
|
85 |
|
public BoundingBox getBoundingbox(short postnr) { |
86 |
|
BoundingBox bb = bbCache.get(postnr); |
87 |
|
return bb.clone();//never return the original / cached object |
88 |
} |
} |
89 |
|
|
90 |
public Queue<Adresse> hentIkkedaekkedeAdresser(int postnr) throws SQLException { |
|
91 |
ConcurrentLinkedQueue<Adresse> queue = new ConcurrentLinkedQueue<Adresse>(); |
public Set<Short> hentPostnumreCache() { |
92 |
|
return postnumre; |
93 |
|
} |
94 |
|
|
95 |
|
|
96 |
|
public Queue<Adresse> hentAlleIkkedaekkedeAdresser(int minPostnr, int maxPostnr) throws SQLException { |
97 |
|
|
98 |
String sql = "SELECT id,postnr,adresse,gadeid,husnr,husnrbogstav,latitude,longitude,rute FROM fulddaekning.adressetabel " + |
logger.info("Henter alle IKKE-daekkede adresser"); |
99 |
|
|
100 |
|
String sql = "SELECT id,a.postnr,adresse,gadeid,husnr,husnrbogstav,latitude,longitude,rute,p.distributor as ho " + |
101 |
|
"FROM fulddaekning.adressetabel a " + |
102 |
|
"LEFT JOIN bogleveringer.postnummerdistributor p on (a.postnr=p.postnr) " + |
103 |
"WHERE rute IS NULL " + //Ingen dækning |
"WHERE rute IS NULL " + //Ingen dækning |
104 |
"AND postnr=? " + |
"AND a.postnr BETWEEN ? AND ? " + |
105 |
"AND latitude IS NOT NULL " + |
"AND latitude IS NOT NULL " + |
106 |
"AND longitude IS NOT NULL " + |
"AND longitude IS NOT NULL " + |
107 |
"AND gadeid IS NOT NULL "; |
"AND gadeid IS NOT NULL " + |
108 |
PreparedStatement stmt = conn.prepareStatement(sql); |
"AND (a.distributor IS NULL OR a.distributor<>'LUKKET') " |
109 |
stmt.setInt(1, postnr); |
//"LIMIT 2500" |
110 |
|
; |
|
queue.addAll( hentAdresseListe( stmt ) ); |
|
|
return queue; |
|
|
} |
|
|
|
|
|
public List<Integer> hentPostnumre() throws SQLException { |
|
|
ArrayList<Integer> list = new ArrayList<Integer>(); |
|
111 |
|
|
112 |
|
if (LookupMain.test_mode == true) { |
113 |
|
sql = sql + " LIMIT 100 "; |
114 |
|
} |
115 |
|
|
|
String sql = "SELECT postnr FROM fulddaekning.adressetabel WHERE distributor = 'DAO' and rute is not null GROUP BY postnr ORDER by postnr"; |
|
116 |
PreparedStatement stmt = conn.prepareStatement(sql); |
PreparedStatement stmt = conn.prepareStatement(sql); |
117 |
ResultSet res = stmt.executeQuery(); |
stmt.setInt(1, minPostnr); |
118 |
|
stmt.setInt(2, maxPostnr); |
119 |
while (res.next()) { |
|
120 |
int postnr = res.getInt("postnr"); |
List<Adresse> list = hentAdresseListe( stmt ); |
121 |
list.add(postnr); |
alleIkkeDaekkede = list.toArray( new Adresse[ list.size() ] ); |
|
} |
|
|
res.close(); |
|
|
stmt.close(); |
|
122 |
|
|
123 |
//list.add(8700); |
logger.info("Analyserer ikke-daekkede adresser"); |
124 |
|
|
125 |
return list; |
for (Adresse a : alleIkkeDaekkede) { |
126 |
|
|
127 |
|
|
128 |
|
BoundingBox bbox; |
129 |
|
|
130 |
|
if (! postnumre.contains(a.postnr )) { |
131 |
|
postnumre.add( a.postnr ); |
132 |
|
|
133 |
|
bbox = new BoundingBox(); |
134 |
|
|
135 |
|
bbCache.put( a.postnr, bbox); |
136 |
|
|
137 |
|
} else { |
138 |
|
bbox = bbCache.get( a.postnr); |
139 |
|
} |
140 |
|
|
141 |
|
bbox.latitudeMax = Math.max(bbox.latitudeMax, a.latitude); |
142 |
|
bbox.latitudeMin = Math.min(bbox.latitudeMin, a.latitude); |
143 |
|
bbox.longitudeMax = Math.max(bbox.longitudeMax, a.longitude); |
144 |
|
bbox.longitudeMin = Math.min(bbox.longitudeMin, a.longitude); |
145 |
|
|
146 |
|
|
147 |
|
} |
148 |
|
return new ConcurrentLinkedQueue<Adresse>( list ); |
149 |
} |
} |
150 |
|
|
151 |
public ArrayList<Adresse> hentDaekkedeAdresser( BoundingBox bbox) throws SQLException { |
|
152 |
String sql = "SELECT id,postnr,adresse,gadeid,husnr,husnrbogstav,latitude,longitude,rute FROM fulddaekning.adressetabel " + |
|
153 |
"WHERE rute IS NOT NULL " + |
|
154 |
"AND latitude BETWEEN ? AND ? " + |
public Map<Short, List<Adresse>> getDaekkedeAdresserHO() { |
155 |
"AND longitude BETWEEN ? AND ? " + |
return daekkedeAdresserHO; |
156 |
"AND distributor = 'DAO' "; |
} |
157 |
|
|
158 |
// Forward only + concur_read_only + fetchsize tvinger driver til at hente en række af gangen (bedre performance ved store result sets) |
public int hentAlleDaekkedeAdresser() throws SQLException { |
159 |
// Se http://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html |
if ( alleAdresser == null ) { |
160 |
//PreparedStatement stmt = conn.prepareStatement(sql); |
String sql = "SELECT id,a.postnr,adresse,gadeid,husnr,husnrbogstav,latitude,longitude,rute,p.distributor as ho " + |
161 |
PreparedStatement stmt = conn.prepareStatement(sql, java.sql.ResultSet.TYPE_FORWARD_ONLY, java.sql.ResultSet.CONCUR_READ_ONLY); |
"FROM fulddaekning.adressetabel a " + |
162 |
stmt.setFetchSize(Integer.MIN_VALUE); |
"LEFT JOIN bogleveringer.postnummerdistributor p on (a.postnr=p.postnr) " + |
163 |
|
"WHERE rute IS NOT NULL " + |
164 |
stmt.setDouble(1, bbox.latitudeMin); |
"AND latitude IS NOT NULL " + |
165 |
stmt.setDouble(2, bbox.latitudeMax); |
"AND longitude IS NOT NULL " + |
166 |
stmt.setDouble(3, bbox.longitudeMin); |
"AND a.distributor = ? "; |
167 |
stmt.setDouble(4, bbox.longitudeMax); |
|
168 |
|
// Forward only + concur_read_only + fetchsize tvinger driver til at hente en række af gangen (bedre performance ved store result sets) |
169 |
return hentAdresseListe( stmt ); |
// Se http://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html |
170 |
|
PreparedStatement stmt = conn.prepareStatement(sql, java.sql.ResultSet.TYPE_FORWARD_ONLY, java.sql.ResultSet.CONCUR_READ_ONLY); |
171 |
|
stmt.setFetchSize(Integer.MIN_VALUE); |
172 |
|
|
173 |
|
stmt.setString(1, LookupMain.distributor); |
174 |
|
|
175 |
|
List<Adresse> list = hentAdresseListe( stmt ); |
176 |
|
|
177 |
|
alleAdresser = new ArrayList<Adresse>(); |
178 |
|
alleAdresser.addAll(list); |
179 |
|
|
180 |
|
for(Adresse addr : list) { |
181 |
|
short ho = addr.ho; |
182 |
|
List<Adresse> hoListe = daekkedeAdresserHO.get(ho); |
183 |
|
if ( hoListe == null) { |
184 |
|
hoListe = new ArrayList<Adresse>(); |
185 |
|
daekkedeAdresserHO.put(ho, hoListe); |
186 |
|
} |
187 |
|
|
188 |
|
hoListe.add(addr); |
189 |
|
|
190 |
|
} |
191 |
|
} |
192 |
|
|
193 |
|
return alleAdresser.size(); |
194 |
} |
} |
|
|
|
195 |
|
|
196 |
|
|
197 |
|
|
198 |
public synchronized void gemResultat(Adresse orgAdresse, Adresse bedsteAdresse, double bedsteAfstand) throws SQLException { |
public synchronized void gemResultat(Adresse orgAdresse, Adresse bedsteAdresse, double bedsteAfstand) throws SQLException { |
199 |
/*String sql = "INSERT INTO fulddaekning.afstand_anden_rute_thn (orgId,orgPostnr, orgAdresse,orgGadeid,orgHusnr,orgHusnrBogstav,orgLatitude,orgLongitude,orgRute,id,postnr,adresse,gadeid,husnr,husnrbogstav,latitude,longitude,rute,afstand,`timestamp`) "+ |
/*String sql = "INSERT INTO fulddaekning.afstand_anden_rute_thn (orgId,orgPostnr, orgAdresse,orgGadeid,orgHusnr,orgHusnrBogstav,orgLatitude,orgLongitude,orgRute,id,postnr,adresse,gadeid,husnr,husnrbogstav,latitude,longitude,rute,afstand,`timestamp`) "+ |
200 |
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )"; |
"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )"; |
201 |
|
|
202 |
PreparedStatement saveStmt = conn.prepareStatement(sql);*/ |
PreparedStatement saveStmt = conn.prepareStatement(sql);*/ |
203 |
|
|
204 |
saveStmt.setInt(1, orgAdresse.id); |
saveStmt.setInt(1, orgAdresse.id); |
205 |
saveStmt.setInt(2, orgAdresse.postnr); |
saveStmt.setShort(2, orgAdresse.postnr); |
206 |
saveStmt.setString(3, orgAdresse.adresse); |
saveStmt.setString(3, orgAdresse.adresse); |
207 |
saveStmt.setInt(4, orgAdresse.gadeid); |
saveStmt.setInt(4, orgAdresse.gadeid); |
208 |
saveStmt.setInt(5, orgAdresse.husnr); |
saveStmt.setShort(5, orgAdresse.husnr); |
209 |
saveStmt.setString(6, orgAdresse.husnrbogstav); |
saveStmt.setString(6, orgAdresse.husnrbogstav); |
210 |
saveStmt.setDouble(7, orgAdresse.latitude); |
saveStmt.setString(7, orgAdresse.rute); |
211 |
saveStmt.setDouble(8, orgAdresse.longitude); |
|
212 |
saveStmt.setString(9, orgAdresse.rute); |
|
213 |
|
saveStmt.setInt(8, bedsteAdresse.id); |
214 |
|
saveStmt.setShort(9, bedsteAdresse.postnr); |
215 |
saveStmt.setInt(10, bedsteAdresse.id); |
saveStmt.setString(10, bedsteAdresse.adresse); |
216 |
saveStmt.setInt(11, bedsteAdresse.postnr); |
saveStmt.setInt(11, bedsteAdresse.gadeid); |
217 |
saveStmt.setString(12, bedsteAdresse.adresse); |
saveStmt.setShort(12, bedsteAdresse.husnr); |
218 |
saveStmt.setInt(13, bedsteAdresse.gadeid); |
saveStmt.setString(13, bedsteAdresse.husnrbogstav); |
219 |
saveStmt.setInt(14, bedsteAdresse.husnr); |
saveStmt.setString(14, bedsteAdresse.rute); |
220 |
saveStmt.setString(15, bedsteAdresse.husnrbogstav); |
|
221 |
saveStmt.setDouble(16, bedsteAdresse.latitude); |
saveStmt.setDouble(15, bedsteAfstand); |
222 |
saveStmt.setDouble(17, bedsteAdresse.longitude); |
|
223 |
saveStmt.setString(18, bedsteAdresse.rute); |
saveStmt.addBatch(); |
224 |
|
batchCount++; |
225 |
saveStmt.setDouble(19, bedsteAfstand); |
if (batchCount >= 1000) { |
226 |
|
saveStmt.executeBatch(); |
227 |
saveStmt.executeUpdate(); |
batchCount = 0; |
228 |
saveStmt.clearParameters(); |
} |
229 |
|
//saveStmt.executeUpdate(); |
230 |
|
//saveStmt.clearParameters(); |
231 |
|
|
232 |
//saveStmt.close(); |
//saveStmt.close(); |
|
|
|
233 |
} |
} |
234 |
|
|
235 |
|
public synchronized void saveBatch() throws SQLException{ |
236 |
|
saveStmt.executeBatch(); |
237 |
|
batchCount = 0; |
238 |
|
} |
239 |
|
|
240 |
|
|
241 |
|
|
242 |
protected ArrayList<Adresse> hentAdresseListe(PreparedStatement stmt) throws SQLException{ |
protected ArrayList<Adresse> hentAdresseListe(PreparedStatement stmt) throws SQLException{ |
243 |
ArrayList<Adresse> list = new ArrayList<Adresse>( 30000 ); |
ArrayList<Adresse> list = new ArrayList<Adresse>( 1000000 ); |
244 |
|
|
245 |
//logger.info("Starting query"); |
//logger.info("Starting query"); |
246 |
ResultSet res = stmt.executeQuery(); |
ResultSet res = stmt.executeQuery(); |
247 |
//logger.info("Starting exec query done"); |
//logger.info("Starting exec query done"); |
248 |
|
|
249 |
while (res.next()) { |
while (res.next()) { |
|
Adresse adr = new Adresse(); |
|
|
adr.id = res.getInt("id"); |
|
|
adr.postnr = res.getInt("postnr"); |
|
|
adr.adresse = res.getString("adresse"); |
|
|
adr.gadeid = res.getInt("gadeid"); |
|
|
adr.husnr = res.getInt("husnr"); |
|
|
adr.husnrbogstav = res.getString("husnrbogstav"); |
|
|
adr.latitude = res.getDouble("latitude"); |
|
|
adr.longitude = res.getDouble("longitude"); |
|
|
adr.rute = res.getString("rute"); |
|
250 |
|
|
251 |
list.add(adr); |
double latitude = res.getDouble(7); |
252 |
|
double longitude = res.getDouble(8); |
253 |
|
|
254 |
|
|
255 |
|
Adresse adr = new Adresse(latitude,longitude); |
256 |
|
|
257 |
|
adr.id = res.getInt(1); |
258 |
|
adr.postnr = res.getShort(2); |
259 |
|
adr.adresse = res.getString(3); |
260 |
|
adr.gadeid = res.getInt(4); |
261 |
|
adr.husnr = res.getShort(5); |
262 |
|
adr.husnrbogstav = husnrbogstavCache.getInstance( res.getString(6) ); |
263 |
|
//adr.latitude = res.getDouble(7); |
264 |
|
//adr.longitude = res.getDouble(8); |
265 |
|
adr.rute = ruteCache.getInstance( res.getString(9) ); |
266 |
|
adr.ho = res.getShort(10); |
267 |
|
|
268 |
|
list.add(adr); |
269 |
|
|
270 |
//logger.info( "Adress:" + adr); |
//logger.info( "Adress:" + adr); |
271 |
} |
} |
272 |
|
|
273 |
res.close(); |
res.close(); |
274 |
stmt.close(); |
stmt.close(); |
275 |
|
|
276 |
return list; |
return list; |
277 |
} |
} |
278 |
|
|
279 |
|
public Connection getConnection(SafeProperties conf) throws SQLException, IOException { |
280 |
|
|
281 |
|
String db_host = conf.getSafeProperty("DB_HOST"); |
282 |
|
String db_user = conf.getSafeProperty("DB_USER"); |
283 |
|
String db_pass = conf.getSafeProperty("DB_PASS"); |
284 |
|
|
285 |
|
|
286 |
|
|
287 |
|
|
288 |
|
Connection conn = null; |
289 |
|
Properties connectionProps = new Properties(); |
290 |
|
connectionProps.put("user", db_user); |
291 |
|
connectionProps.put("password", db_pass); |
292 |
|
|
293 |
|
//For debug output, tilføj denne til JDBC url'en: &profileSQL=true |
294 |
|
conn = DriverManager.getConnection( |
295 |
|
"jdbc:mysql://" + |
296 |
|
db_host + |
297 |
|
":3306/?rewriteBatchedStatements=true", |
298 |
|
connectionProps); |
299 |
|
logger.info("Connected to database"); |
300 |
|
return conn; |
301 |
|
} |
302 |
|
|
303 |
|
|
304 |
} |
} |