/[projects]/dao/DaoAdresseVedligehold/src/main/java/dk/daoas/adressevedligehold/db/DatabaseCoverageUpdate.java
ViewVC logotype

Contents of /dao/DaoAdresseVedligehold/src/main/java/dk/daoas/adressevedligehold/db/DatabaseCoverageUpdate.java

Parent Directory Parent Directory | Revision Log Revision Log


Revision 3023 - (show annotations) (download)
Tue Apr 26 18:57:25 2016 UTC (8 years ago) by torben
File size: 10011 byte(s)
Use batch inserts with on duplicate key update, in order to speedup updates
1 package dk.daoas.adressevedligehold.db;
2
3 import java.sql.Connection;
4 import java.sql.PreparedStatement;
5 import java.sql.ResultSet;
6 import java.sql.SQLException;
7 import java.sql.Statement;
8 import java.util.ArrayList;
9 import java.util.List;
10
11
12
13 import dk.daoas.adressevedligehold.beans.Address;
14 import dk.daoas.adressevedligehold.beans.Address.AddressState;
15 import dk.daoas.adressevedligehold.tasks.TaskLogger;
16 import dk.daoas.adressevedligehold.util.DeduplicateHelper;
17 import dk.daoas.adressevedligehold.util.TimingHelper;
18
19 /*
20 *
21 */
22
23 public class DatabaseCoverageUpdate {
24
25 //static boolean DEBUG = false;
26
27 private TaskLogger logger = TaskLogger.getInstance();
28
29
30 public List<Address> getAllAdresses() throws SQLException {
31 TimingHelper timing = new TimingHelper();
32
33 //String debugFilter = DatabaseLayerImplementation.DEBUG ? " WHERE postnr >= 6000 " : "";
34
35 String sql =
36 "SELECT id,vejnavn,husnr,husnrbogstav,kommunekode,vejkode,postnr,gadeid,upper(distributor) AS distributor,dbkbane,latitude,longitude, "
37 + "rutema,korelistema,ruteti,korelisteti,ruteon,korelisteon,ruteto,korelisteto,rutefr,korelistefr,rutelo,korelistelo,ruteso,korelisteso "
38 + "FROM fulddaekning.adressetabel "
39 //+ debugFilter
40 ;
41
42 try ( Connection conn = DBConnection.getConnection();
43 Statement stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
44 ) {
45 stmt.setFetchSize(Integer.MIN_VALUE);
46 ResultSet res = stmt.executeQuery(sql);
47
48 List<Address> list = new ArrayList<Address>(2600000);//initial capacity 2.6 mio
49
50 DeduplicateHelper<String> vejnavnCache = new DeduplicateHelper<String>(81920);
51 DeduplicateHelper<String> husnrbogstavCache = new DeduplicateHelper<String>();
52 DeduplicateHelper<String> distributorCache = new DeduplicateHelper<String>();
53 DeduplicateHelper<String> dirigeringsCache = new DeduplicateHelper<String>(16*1024);
54
55
56 while (res.next()) {
57
58 Address a = new Address();
59 a.id = res.getInt(1);
60 a.vejnavn = vejnavnCache.getInstance( res.getString(2) );
61 a.husnr = (short) res.getInt(3);
62 a.husnrbogstav = husnrbogstavCache.getInstance( res.getString(4) );
63 a.kommunekode = (short) res.getInt(5);
64 a.vejkode = (short)res.getInt(6);
65 a.postnr = (short)res.getInt(7);
66 a.gadeid = res.getInt(8);
67 a.distributor = distributorCache.getInstance(res.getString(9));
68 a.dbkBane = (short) res.getInt(10);
69 a.latitude = (float) res.getDouble(11);
70 a.longitude = (float) res.getDouble(12);
71
72 a.ruteMandag = dirigeringsCache.getInstance( res.getString(13) );
73 a.korelisteMandag = dirigeringsCache.getInstance( res.getString(14) );
74
75 a.ruteTirsdag = dirigeringsCache.getInstance( res.getString(15) );
76 a.korelisteTirsdag = dirigeringsCache.getInstance( res.getString(16) );
77
78 a.ruteOnsdag = dirigeringsCache.getInstance( res.getString(17) );
79 a.korelisteOnsdag = dirigeringsCache.getInstance( res.getString(18) );
80
81 a.ruteTorsdag = dirigeringsCache.getInstance( res.getString(19) );
82 a.korelisteTorsdag = dirigeringsCache.getInstance( res.getString(20) );
83
84 a.ruteFredag = dirigeringsCache.getInstance( res.getString(21) );
85 a.korelisteFredag = dirigeringsCache.getInstance( res.getString(22) );
86
87 a.ruteLordag = dirigeringsCache.getInstance( res.getString(23) );
88 a.korelisteLordag = dirigeringsCache.getInstance( res.getString(24) );
89
90 a.ruteSondag = dirigeringsCache.getInstance( res.getString(25) );
91 a.korelisteSondag = dirigeringsCache.getInstance( res.getString(26) );
92
93
94 list.add(a);
95 }
96 res.close();
97
98
99 logger.info("DB Loaded " + list.size() + " adresses in " + timing.getElapsed() + "ms");
100
101 return list;
102 }
103 }
104
105 public void saveNewAddresses(List<Address> addresses) {
106
107 logger.info("DB::saveNewAddresses size:" + addresses.size() );
108
109 String sql = "INSERT IGNORE INTO fulddaekning.adressetabel " +
110 "(vejnavn,husnr,husnrbogstav,kommunekode,vejkode,postnr,gadeid,dbkbane,indlast,aendret,david,googlechecked," +
111 "RuteMa,KorelisteMa,RuteTi,KorelisteTi,RuteOn,KorelisteOn,RuteTo,KorelisteTo,RuteFr,KorelisteFr,RuteLo,KorelisteLo,RuteSo,KorelisteSo, " +
112 "kommentar,latitude,longitude) " +
113 "VALUES " +
114 "(?,?,?,?,?,?,?,?, now(), now(), ?, 0, " +
115 "?,?,?,?,?,?,?,?,?,?,?,?,?,?," +
116 "?,?,?) ";
117
118
119 try (Connection con = DBConnection.getConnection()) {
120
121
122 try (PreparedStatement stmt = con.prepareStatement(sql)) {
123 for (Address addr : addresses) {
124
125 String david = String.format("%04d%03d%04d", addr.postnr, addr.kommunekode, addr.vejkode);
126
127 stmt.setString(1, addr.vejnavn);
128 stmt.setShort(2, addr.husnr);
129 stmt.setString(3, addr.husnrbogstav);
130 stmt.setShort(4, addr.kommunekode);
131 stmt.setShort(5, addr.vejkode);
132 stmt.setShort(6, addr.postnr);
133 stmt.setInt(7, addr.gadeid);
134 stmt.setShort(8, addr.dbkBane);
135 stmt.setString(9, david);
136
137 stmt.setString(10, addr.ruteMandag);
138 stmt.setString(11, addr.korelisteMandag);
139 stmt.setString(12, addr.ruteTirsdag);
140 stmt.setString(13, addr.korelisteTirsdag);
141 stmt.setString(14, addr.ruteOnsdag);
142 stmt.setString(15, addr.korelisteOnsdag);
143 stmt.setString(16, addr.ruteTorsdag);
144 stmt.setString(17, addr.korelisteTorsdag);
145 stmt.setString(18, addr.ruteFredag);
146 stmt.setString(19, addr.korelisteFredag);
147 stmt.setString(20, addr.ruteLordag);
148 stmt.setString(21, addr.korelisteLordag);
149 stmt.setString(22, addr.ruteSondag);
150 stmt.setString(23, addr.korelisteSondag);
151
152 stmt.setString(24, addr.kommentar);
153
154 if (addr.latitude != 0.0) {
155 stmt.setDouble(25, addr.latitude);
156 } else {
157 stmt.setNull(25, java.sql.Types.DOUBLE);
158 }
159 if (addr.longitude != 0.0) {
160 stmt.setDouble(26, addr.longitude);
161 } else {
162 stmt.setNull(26, java.sql.Types.DOUBLE);
163 }
164
165
166 stmt.execute();
167 if (stmt.getUpdateCount() != 1) {
168 logger.info("Error executing insert: " + addr);
169 addr.state = AddressState.CREATE_FAIL;
170 }
171
172 }
173 }
174
175 //con.commit();
176 } catch (SQLException e) {
177 logger.warning("Error inserting addresses", e );
178 }
179 }
180
181 public void updateAddresses(List<Address> addresses) {
182 logger.info("DB::UpdateAddresses size:" + addresses.size() );
183
184 /*String sql = "UPDATE fulddaekning.adressetabel " +
185 "SET " +
186 "RuteMa=?, KorelisteMa=?, " +
187 "RuteTi=?, KorelisteTi=?, " +
188 "RuteOn=?, KorelisteOn=?, " +
189 "RuteTo=?, KorelisteTo=?, " +
190 "RuteFr=?, KorelisteFr=?, " +
191 "RuteLo=?, KorelisteLo=?, " +
192 "RuteSo=?, KorelisteSo=?, " +
193 "Distributor=?, " +
194 "DBKBane=?, " +
195 "Aendret=now() " +
196 "WHERE id=? ";*/
197 String sql = "INSERT INTO fulddaekning.adressetabel " +
198 "(id, " +
199 "RuteMa, KorelisteMa, RuteTi, KorelisteTi, RuteOn, KorelisteOn, " +
200 "RuteTo, KorelisteTo, RuteFr, KorelisteFr, RuteLo, KorelisteLo, " +
201 "RuteSo, KorelisteSo, " +
202 "Distributor, DBKBane, Aendret, " +
203 "vejnavn,husnr,husnrbogstav,kommunekode,vejkode,gadeid,postnr) \n" + //Disse har ingen default value
204 "VALUES " +
205 "(?," +
206 "?,?,?,?,?,?," +
207 "?,?,?,?,?,?," +
208 "?,?," +
209 "?,?, now(), " +
210 "'',0,'',0,0,0,0 ) \n" +
211
212 "ON DUPLICATE KEY UPDATE " +
213 "RuteMa=VALUES(RuteMa), KorelisteMa=VALUES(KorelisteMa), " +
214 "RuteTi=VALUES(RuteTi), KorelisteTi=VALUES(KorelisteTi), " +
215 "RuteOn=VALUES(RuteOn), KorelisteOn=VALUES(KorelisteOn), " +
216 "RuteTo=VALUES(RuteTo), KorelisteTo=VALUES(KorelisteTo), " +
217 "RuteFr=VALUES(RuteFr), KorelisteFr=VALUES(KorelisteFr), " +
218 "RuteLo=VALUES(RuteLo), KorelisteLo=VALUES(KorelisteLo), " +
219 "RuteSo=VALUES(RuteSo), KorelisteSo=VALUES(KorelisteSo), " +
220 "Distributor=VALUES(Distributor), " +
221 "DBKBane=VALUES(DBKBane) ";
222
223
224
225
226 try (Connection con = DBConnection.getConnection()) {
227 //con.setAutoCommit(false);
228
229 try (PreparedStatement stmt = con.prepareStatement(sql)) {
230 int count = 0;
231
232 for (Address addr : addresses) {
233 stmt.setInt(1, addr.id);
234 stmt.setString(2, addr.ruteMandag);
235 stmt.setString(3, addr.korelisteMandag);
236 stmt.setString(4, addr.ruteTirsdag);
237 stmt.setString(5, addr.korelisteTirsdag);
238 stmt.setString(6, addr.ruteOnsdag);
239 stmt.setString(7, addr.korelisteOnsdag);
240 stmt.setString(8, addr.ruteTorsdag);
241 stmt.setString(9, addr.korelisteTorsdag);
242 stmt.setString(10, addr.ruteFredag);
243 stmt.setString(11, addr.korelisteFredag);
244 stmt.setString(12, addr.ruteLordag);
245 stmt.setString(13, addr.korelisteLordag);
246 stmt.setString(14, addr.ruteSondag);
247 stmt.setString(15, addr.korelisteSondag);
248
249 stmt.setString(16, addr.distributor);
250 stmt.setInt(17, addr.dbkBane);
251
252
253
254 stmt.addBatch();
255 count++;
256 if ( (count%200) == 0) {
257 stmt.executeBatch();
258
259 if ( (count%50000) == 0 ) {
260 logger.info("Update count " + count);
261 }
262 }
263 }
264
265 stmt.executeBatch(); //update remainding queries
266
267 }
268
269 //con.commit();
270 } catch (SQLException e) {
271 logger.warning("Error updating addresses", e );
272 }
273 }
274
275 /*
276 private static String nullify(String str) {
277 if (str == null)
278 return null;
279
280 if (str.equals("")) {
281 return null;
282 } else {
283 return str;
284 }
285 }
286
287
288 /*
289 private static int safeInt(String str) {
290 try {
291 return Integer.parseInt( str );
292 } catch (NumberFormatException e) {
293 return 0;
294 }
295 }
296
297
298
299 private static String coalesce(String s1, String s2) {
300 if (s1 != null)
301 return s1;
302
303 return s2;
304 }
305
306 */
307 }

  ViewVC Help
Powered by ViewVC 1.1.20