/[projects]/dao/DaoAdresseVedligehold/src/main/java/dk/daoas/adressevedligehold/db/DatabaseCoverageUpdate.java
ViewVC logotype

Annotation of /dao/DaoAdresseVedligehold/src/main/java/dk/daoas/adressevedligehold/db/DatabaseCoverageUpdate.java

Parent Directory Parent Directory | Revision Log Revision Log


Revision 3023 - (hide annotations) (download)
Tue Apr 26 18:57:25 2016 UTC (8 years, 1 month ago) by torben
File size: 10011 byte(s)
Use batch inserts with on duplicate key update, in order to speedup updates
1 torben 2838 package dk.daoas.adressevedligehold.db;
2    
3     import java.sql.Connection;
4     import java.sql.PreparedStatement;
5     import java.sql.ResultSet;
6     import java.sql.SQLException;
7     import java.sql.Statement;
8     import java.util.ArrayList;
9     import java.util.List;
10    
11 torben 2876
12 torben 2903
13 torben 2838 import dk.daoas.adressevedligehold.beans.Address;
14 torben 2876 import dk.daoas.adressevedligehold.beans.Address.AddressState;
15 torben 2903 import dk.daoas.adressevedligehold.tasks.TaskLogger;
16 torben 2838 import dk.daoas.adressevedligehold.util.DeduplicateHelper;
17     import dk.daoas.adressevedligehold.util.TimingHelper;
18    
19 torben 2950 /*
20 torben 2840 *
21     */
22 torben 2838
23 torben 2946 public class DatabaseCoverageUpdate {
24 torben 2838
25 torben 2847 //static boolean DEBUG = false;
26 torben 2838
27 torben 2903 private TaskLogger logger = TaskLogger.getInstance();
28 torben 2838
29 torben 2903
30 torben 2838 public List<Address> getAllAdresses() throws SQLException {
31     TimingHelper timing = new TimingHelper();
32    
33 torben 2847 //String debugFilter = DatabaseLayerImplementation.DEBUG ? " WHERE postnr >= 6000 " : "";
34 torben 2838
35     String sql =
36     "SELECT id,vejnavn,husnr,husnrbogstav,kommunekode,vejkode,postnr,gadeid,upper(distributor) AS distributor,dbkbane,latitude,longitude, "
37     + "rutema,korelistema,ruteti,korelisteti,ruteon,korelisteon,ruteto,korelisteto,rutefr,korelistefr,rutelo,korelistelo,ruteso,korelisteso "
38     + "FROM fulddaekning.adressetabel "
39 torben 2847 //+ debugFilter
40 torben 2838 ;
41    
42     try ( Connection conn = DBConnection.getConnection();
43 torben 2937 Statement stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
44 torben 2838 ) {
45     stmt.setFetchSize(Integer.MIN_VALUE);
46     ResultSet res = stmt.executeQuery(sql);
47    
48     List<Address> list = new ArrayList<Address>(2600000);//initial capacity 2.6 mio
49    
50     DeduplicateHelper<String> vejnavnCache = new DeduplicateHelper<String>(81920);
51     DeduplicateHelper<String> husnrbogstavCache = new DeduplicateHelper<String>();
52     DeduplicateHelper<String> distributorCache = new DeduplicateHelper<String>();
53     DeduplicateHelper<String> dirigeringsCache = new DeduplicateHelper<String>(16*1024);
54    
55    
56     while (res.next()) {
57    
58     Address a = new Address();
59     a.id = res.getInt(1);
60     a.vejnavn = vejnavnCache.getInstance( res.getString(2) );
61     a.husnr = (short) res.getInt(3);
62     a.husnrbogstav = husnrbogstavCache.getInstance( res.getString(4) );
63     a.kommunekode = (short) res.getInt(5);
64     a.vejkode = (short)res.getInt(6);
65     a.postnr = (short)res.getInt(7);
66     a.gadeid = res.getInt(8);
67     a.distributor = distributorCache.getInstance(res.getString(9));
68     a.dbkBane = (short) res.getInt(10);
69     a.latitude = (float) res.getDouble(11);
70     a.longitude = (float) res.getDouble(12);
71    
72     a.ruteMandag = dirigeringsCache.getInstance( res.getString(13) );
73     a.korelisteMandag = dirigeringsCache.getInstance( res.getString(14) );
74    
75     a.ruteTirsdag = dirigeringsCache.getInstance( res.getString(15) );
76     a.korelisteTirsdag = dirigeringsCache.getInstance( res.getString(16) );
77    
78     a.ruteOnsdag = dirigeringsCache.getInstance( res.getString(17) );
79     a.korelisteOnsdag = dirigeringsCache.getInstance( res.getString(18) );
80    
81     a.ruteTorsdag = dirigeringsCache.getInstance( res.getString(19) );
82     a.korelisteTorsdag = dirigeringsCache.getInstance( res.getString(20) );
83    
84     a.ruteFredag = dirigeringsCache.getInstance( res.getString(21) );
85     a.korelisteFredag = dirigeringsCache.getInstance( res.getString(22) );
86    
87     a.ruteLordag = dirigeringsCache.getInstance( res.getString(23) );
88     a.korelisteLordag = dirigeringsCache.getInstance( res.getString(24) );
89    
90     a.ruteSondag = dirigeringsCache.getInstance( res.getString(25) );
91     a.korelisteSondag = dirigeringsCache.getInstance( res.getString(26) );
92    
93    
94     list.add(a);
95     }
96     res.close();
97    
98    
99 torben 2903 logger.info("DB Loaded " + list.size() + " adresses in " + timing.getElapsed() + "ms");
100 torben 2838
101     return list;
102     }
103     }
104    
105 torben 2876 public void saveNewAddresses(List<Address> addresses) {
106 torben 3023
107 torben 2903 logger.info("DB::saveNewAddresses size:" + addresses.size() );
108 torben 2840
109 torben 2876 String sql = "INSERT IGNORE INTO fulddaekning.adressetabel " +
110     "(vejnavn,husnr,husnrbogstav,kommunekode,vejkode,postnr,gadeid,dbkbane,indlast,aendret,david,googlechecked," +
111 torben 3003 "RuteMa,KorelisteMa,RuteTi,KorelisteTi,RuteOn,KorelisteOn,RuteTo,KorelisteTo,RuteFr,KorelisteFr,RuteLo,KorelisteLo,RuteSo,KorelisteSo, " +
112     "kommentar,latitude,longitude) " +
113 torben 2876 "VALUES " +
114     "(?,?,?,?,?,?,?,?, now(), now(), ?, 0, " +
115 torben 3003 "?,?,?,?,?,?,?,?,?,?,?,?,?,?," +
116     "?,?,?) ";
117 torben 2876
118    
119     try (Connection con = DBConnection.getConnection()) {
120 torben 2950
121 torben 2876
122     try (PreparedStatement stmt = con.prepareStatement(sql)) {
123     for (Address addr : addresses) {
124    
125     String david = String.format("%04d%03d%04d", addr.postnr, addr.kommunekode, addr.vejkode);
126    
127     stmt.setString(1, addr.vejnavn);
128     stmt.setShort(2, addr.husnr);
129     stmt.setString(3, addr.husnrbogstav);
130     stmt.setShort(4, addr.kommunekode);
131     stmt.setShort(5, addr.vejkode);
132     stmt.setShort(6, addr.postnr);
133     stmt.setInt(7, addr.gadeid);
134     stmt.setShort(8, addr.dbkBane);
135     stmt.setString(9, david);
136    
137     stmt.setString(10, addr.ruteMandag);
138     stmt.setString(11, addr.korelisteMandag);
139     stmt.setString(12, addr.ruteTirsdag);
140     stmt.setString(13, addr.korelisteTirsdag);
141     stmt.setString(14, addr.ruteOnsdag);
142     stmt.setString(15, addr.korelisteOnsdag);
143     stmt.setString(16, addr.ruteTorsdag);
144     stmt.setString(17, addr.korelisteTorsdag);
145     stmt.setString(18, addr.ruteFredag);
146     stmt.setString(19, addr.korelisteFredag);
147     stmt.setString(20, addr.ruteLordag);
148     stmt.setString(21, addr.korelisteLordag);
149     stmt.setString(22, addr.ruteSondag);
150     stmt.setString(23, addr.korelisteSondag);
151 torben 3003
152     stmt.setString(24, addr.kommentar);
153    
154     if (addr.latitude != 0.0) {
155     stmt.setDouble(25, addr.latitude);
156     } else {
157     stmt.setNull(25, java.sql.Types.DOUBLE);
158     }
159     if (addr.longitude != 0.0) {
160     stmt.setDouble(26, addr.longitude);
161     } else {
162     stmt.setNull(26, java.sql.Types.DOUBLE);
163     }
164 torben 2876
165    
166     stmt.execute();
167     if (stmt.getUpdateCount() != 1) {
168 torben 2903 logger.info("Error executing insert: " + addr);
169 torben 2876 addr.state = AddressState.CREATE_FAIL;
170     }
171    
172     }
173     }
174    
175     //con.commit();
176     } catch (SQLException e) {
177 torben 2903 logger.warning("Error inserting addresses", e );
178 torben 2876 }
179 torben 2840 }
180    
181 torben 2838 public void updateAddresses(List<Address> addresses) {
182 torben 2903 logger.info("DB::UpdateAddresses size:" + addresses.size() );
183 torben 2838
184 torben 3023 /*String sql = "UPDATE fulddaekning.adressetabel " +
185 torben 2838 "SET " +
186     "RuteMa=?, KorelisteMa=?, " +
187     "RuteTi=?, KorelisteTi=?, " +
188     "RuteOn=?, KorelisteOn=?, " +
189     "RuteTo=?, KorelisteTo=?, " +
190     "RuteFr=?, KorelisteFr=?, " +
191     "RuteLo=?, KorelisteLo=?, " +
192 torben 2845 "RuteSo=?, KorelisteSo=?, " +
193 torben 2847 "Distributor=?, " +
194 torben 2853 "DBKBane=?, " +
195     "Aendret=now() " +
196 torben 3023 "WHERE id=? ";*/
197     String sql = "INSERT INTO fulddaekning.adressetabel " +
198     "(id, " +
199     "RuteMa, KorelisteMa, RuteTi, KorelisteTi, RuteOn, KorelisteOn, " +
200     "RuteTo, KorelisteTo, RuteFr, KorelisteFr, RuteLo, KorelisteLo, " +
201     "RuteSo, KorelisteSo, " +
202     "Distributor, DBKBane, Aendret, " +
203     "vejnavn,husnr,husnrbogstav,kommunekode,vejkode,gadeid,postnr) \n" + //Disse har ingen default value
204     "VALUES " +
205     "(?," +
206     "?,?,?,?,?,?," +
207     "?,?,?,?,?,?," +
208     "?,?," +
209     "?,?, now(), " +
210     "'',0,'',0,0,0,0 ) \n" +
211 torben 2838
212 torben 3023 "ON DUPLICATE KEY UPDATE " +
213     "RuteMa=VALUES(RuteMa), KorelisteMa=VALUES(KorelisteMa), " +
214     "RuteTi=VALUES(RuteTi), KorelisteTi=VALUES(KorelisteTi), " +
215     "RuteOn=VALUES(RuteOn), KorelisteOn=VALUES(KorelisteOn), " +
216     "RuteTo=VALUES(RuteTo), KorelisteTo=VALUES(KorelisteTo), " +
217     "RuteFr=VALUES(RuteFr), KorelisteFr=VALUES(KorelisteFr), " +
218     "RuteLo=VALUES(RuteLo), KorelisteLo=VALUES(KorelisteLo), " +
219     "RuteSo=VALUES(RuteSo), KorelisteSo=VALUES(KorelisteSo), " +
220     "Distributor=VALUES(Distributor), " +
221     "DBKBane=VALUES(DBKBane) ";
222 torben 2838
223 torben 3023
224    
225 torben 2838
226     try (Connection con = DBConnection.getConnection()) {
227 torben 2845 //con.setAutoCommit(false);
228 torben 2838
229     try (PreparedStatement stmt = con.prepareStatement(sql)) {
230 torben 3018 int count = 0;
231    
232 torben 2838 for (Address addr : addresses) {
233 torben 3023 stmt.setInt(1, addr.id);
234     stmt.setString(2, addr.ruteMandag);
235     stmt.setString(3, addr.korelisteMandag);
236     stmt.setString(4, addr.ruteTirsdag);
237     stmt.setString(5, addr.korelisteTirsdag);
238     stmt.setString(6, addr.ruteOnsdag);
239     stmt.setString(7, addr.korelisteOnsdag);
240     stmt.setString(8, addr.ruteTorsdag);
241     stmt.setString(9, addr.korelisteTorsdag);
242     stmt.setString(10, addr.ruteFredag);
243     stmt.setString(11, addr.korelisteFredag);
244     stmt.setString(12, addr.ruteLordag);
245     stmt.setString(13, addr.korelisteLordag);
246     stmt.setString(14, addr.ruteSondag);
247     stmt.setString(15, addr.korelisteSondag);
248 torben 2838
249 torben 3023 stmt.setString(16, addr.distributor);
250     stmt.setInt(17, addr.dbkBane);
251 torben 2838
252    
253 torben 3023
254 torben 3018 stmt.addBatch();
255     count++;
256 torben 3023 if ( (count%200) == 0) {
257 torben 3018 stmt.executeBatch();
258 torben 3023
259     if ( (count%50000) == 0 ) {
260     logger.info("Update count " + count);
261     }
262 torben 3018 }
263 torben 2838 }
264 torben 3018
265     stmt.executeBatch(); //update remainding queries
266    
267 torben 2838 }
268    
269 torben 2845 //con.commit();
270 torben 2838 } catch (SQLException e) {
271 torben 2903 logger.warning("Error updating addresses", e );
272 torben 2838 }
273     }
274    
275 torben 2898 /*
276 torben 2838 private static String nullify(String str) {
277     if (str == null)
278     return null;
279    
280     if (str.equals("")) {
281     return null;
282     } else {
283     return str;
284     }
285     }
286    
287    
288     /*
289     private static int safeInt(String str) {
290     try {
291     return Integer.parseInt( str );
292     } catch (NumberFormatException e) {
293     return 0;
294     }
295     }
296    
297    
298    
299     private static String coalesce(String s1, String s2) {
300     if (s1 != null)
301     return s1;
302    
303     return s2;
304     }
305    
306     */
307     }

  ViewVC Help
Powered by ViewVC 1.1.20