adressevedligehold/db/DatabaseCoverageUpdate.java

package dk.daoas.adressevedligehold.db;

import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;


import dk.daoas.adressevedligehold.beans.Address;
import dk.daoas.adressevedligehold.beans.Address.AddressState;
import dk.daoas.adressevedligehold.tasks.TaskLogger;
import dk.daoas.adressevedligehold.util.DeduplicateHelper;
import dk.daoas.adressevedligehold.util.TimingHelper;

/* 
 *  
 */

public class DatabaseCoverageUpdate  {
        
        //static boolean DEBUG = false;
        
        private TaskLogger logger = TaskLogger.getInstance();
        
        
        public List<Address> getAllAdresses() throws SQLException {
                TimingHelper timing = new TimingHelper();
                
                //String debugFilter = DatabaseLayerImplementation.DEBUG ? " WHERE postnr >= 6000 " : "";
                
                String sql = 
                                "SELECT id,vejnavn,husnr,husnrbogstav,kommunekode,vejkode,postnr,gadeid,upper(distributor) AS distributor,dbkbane,latitude,longitude, "
                                + "rutema,korelistema,ruteti,korelisteti,ruteon,korelisteon,ruteto,korelisteto,rutefr,korelistefr,rutelo,korelistelo,ruteso,korelisteso "
                                + "FROM fulddaekning.adressetabel "
                                //+ debugFilter
                                ;
                
                try ( Connection conn = DBConnection.getConnection();           
                                Statement stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
                                ) {
                        stmt.setFetchSize(Integer.MIN_VALUE);
                        ResultSet res = stmt.executeQuery(sql);
                        
                        List<Address> list = new ArrayList<Address>(2600000);//initial capacity 2.6 mio
                        
                        DeduplicateHelper<String> vejnavnCache = new DeduplicateHelper<String>(81920);
                        DeduplicateHelper<String> husnrbogstavCache = new DeduplicateHelper<String>();
                        DeduplicateHelper<String> distributorCache = new DeduplicateHelper<String>();
                        DeduplicateHelper<String> dirigeringsCache = new DeduplicateHelper<String>(16*1024);                    
        
                        
                        while (res.next()) {
                                
                                Address a = new Address();
                                a.id = res.getInt(1);
                                a.vejnavn = vejnavnCache.getInstance( res.getString(2) );
                                a.husnr = (short) res.getInt(3);
                                a.husnrbogstav = husnrbogstavCache.getInstance( res.getString(4) );
                                a.kommunekode = (short) res.getInt(5);
                                a.vejkode = (short)res.getInt(6);
                                a.postnr = (short)res.getInt(7);
                                a.gadeid = res.getInt(8);
                                a.distributor = distributorCache.getInstance(res.getString(9));
                                a.dbkBane = (short) res.getInt(10);
                                a.latitude = (float) res.getDouble(11);
                                a.longitude = (float) res.getDouble(12);
                                
                                a.ruteMandag = dirigeringsCache.getInstance( res.getString(13) );
                                a.korelisteMandag = dirigeringsCache.getInstance( res.getString(14) );
                                
                                a.ruteTirsdag = dirigeringsCache.getInstance( res.getString(15) );
                                a.korelisteTirsdag = dirigeringsCache.getInstance( res.getString(16) );                         

                                a.ruteOnsdag = dirigeringsCache.getInstance( res.getString(17) );
                                a.korelisteOnsdag = dirigeringsCache.getInstance( res.getString(18) );
                                
                                a.ruteTorsdag = dirigeringsCache.getInstance( res.getString(19) );
                                a.korelisteTorsdag = dirigeringsCache.getInstance( res.getString(20) );
                                
                                a.ruteFredag = dirigeringsCache.getInstance( res.getString(21) );
                                a.korelisteFredag = dirigeringsCache.getInstance( res.getString(22) );
                                
                                a.ruteLordag = dirigeringsCache.getInstance( res.getString(23) );
                                a.korelisteLordag = dirigeringsCache.getInstance( res.getString(24) );
                                
                                a.ruteSondag = dirigeringsCache.getInstance( res.getString(25) );
                                a.korelisteSondag = dirigeringsCache.getInstance( res.getString(26) );


                                list.add(a);
                        }
                        res.close();
                        
                        
                        logger.info("DB Loaded " + list.size() + " adresses in " + timing.getElapsed() + "ms");
                        
                        return list;
                }
        }
        
        public void saveNewAddresses(List<Address> addresses) {
                logger.info("DB::saveNewAddresses size:" + addresses.size() );
                
                String sql = "INSERT IGNORE INTO fulddaekning.adressetabel " +
                                         "(vejnavn,husnr,husnrbogstav,kommunekode,vejkode,postnr,gadeid,dbkbane,indlast,aendret,david,googlechecked," +
                                         "RuteMa,KorelisteMa,RuteTi,KorelisteTi,RuteOn,KorelisteOn,RuteTo,KorelisteTo,RuteFr,KorelisteFr,RuteLo,KorelisteLo,RuteSo,KorelisteSo, " +
                                         "kommentar,latitude,longitude) " +
                                         "VALUES " +
                                         "(?,?,?,?,?,?,?,?, now(), now(), ?, 0, " +
                                         "?,?,?,?,?,?,?,?,?,?,?,?,?,?," +
                                         "?,?,?) ";
                
                
                try (Connection con = DBConnection.getConnection()) {

                        
                        try (PreparedStatement stmt = con.prepareStatement(sql)) {
                                for (Address addr : addresses) {
                                        
                                        String david = String.format("%04d%03d%04d", addr.postnr, addr.kommunekode, addr.vejkode);
                                        
                                        stmt.setString(1, addr.vejnavn);
                                        stmt.setShort(2, addr.husnr);
                                        stmt.setString(3, addr.husnrbogstav);
                                        stmt.setShort(4, addr.kommunekode);
                                        stmt.setShort(5, addr.vejkode);
                                        stmt.setShort(6, addr.postnr);
                                        stmt.setInt(7, addr.gadeid);
                                        stmt.setShort(8, addr.dbkBane);
                                        stmt.setString(9, david);
                                        
                                        stmt.setString(10, addr.ruteMandag);
                                        stmt.setString(11, addr.korelisteMandag);
                                        stmt.setString(12, addr.ruteTirsdag);
                                        stmt.setString(13, addr.korelisteTirsdag);
                                        stmt.setString(14, addr.ruteOnsdag);
                                        stmt.setString(15, addr.korelisteOnsdag);
                                        stmt.setString(16, addr.ruteTorsdag);
                                        stmt.setString(17, addr.korelisteTorsdag);
                                        stmt.setString(18, addr.ruteFredag);
                                        stmt.setString(19, addr.korelisteFredag);
                                        stmt.setString(20, addr.ruteLordag);
                                        stmt.setString(21, addr.korelisteLordag);
                                        stmt.setString(22, addr.ruteSondag);
                                        stmt.setString(23, addr.korelisteSondag);
                                        
                                        stmt.setString(24, addr.kommentar);
                                        
                                        if (addr.latitude != 0.0) {
                                                stmt.setDouble(25, addr.latitude);
                                        } else {
                                                stmt.setNull(25, java.sql.Types.DOUBLE);
                                        }                                       
                                        if (addr.longitude != 0.0) {
                                                stmt.setDouble(26, addr.longitude);
                                        } else {
                                                stmt.setNull(26, java.sql.Types.DOUBLE);
                                        }

                                        
                                        stmt.execute();
                                        if (stmt.getUpdateCount() != 1) {
                                                logger.info("Error executing insert: " + addr);
                                                addr.state = AddressState.CREATE_FAIL;
                                        }
                                        
                                }
                        }
                        
                        //con.commit();
                } catch (SQLException e) {
                        logger.warning("Error inserting addresses", e  );
                }
        }
        
        public void updateAddresses(List<Address> addresses) {
                logger.info("DB::UpdateAddresses size:" + addresses.size() );
                
                String sql = "UPDATE fulddaekning.adressetabel " +
                                        "SET " +
                                        "RuteMa=?, KorelisteMa=?, " +
                                        "RuteTi=?, KorelisteTi=?, " +
                                        "RuteOn=?, KorelisteOn=?, " +
                                        "RuteTo=?, KorelisteTo=?, " +
                                        "RuteFr=?, KorelisteFr=?, " +
                                        "RuteLo=?, KorelisteLo=?, " +
                                        "RuteSo=?, KorelisteSo=?, " +
                                    "Distributor=?, " +
                                    "DBKBane=?, " +
                                    "Aendret=now() " +
                                        "WHERE id=? ";
                                        
                                        
                try (Connection con = DBConnection.getConnection()) {
                        //con.setAutoCommit(false);
                        
                        try (PreparedStatement stmt = con.prepareStatement(sql)) {
                                int count = 0;
                                
                                for (Address addr : addresses) {
                                        stmt.setString(1, addr.ruteMandag);
                                        stmt.setString(2, addr.korelisteMandag);
                                        stmt.setString(3, addr.ruteTirsdag);
                                        stmt.setString(4, addr.korelisteTirsdag);
                                        stmt.setString(5, addr.ruteOnsdag);
                                        stmt.setString(6, addr.korelisteOnsdag);
                                        stmt.setString(7, addr.ruteTorsdag);
                                        stmt.setString(8, addr.korelisteTorsdag);
                                        stmt.setString(9, addr.ruteFredag);
                                        stmt.setString(10, addr.korelisteFredag);
                                        stmt.setString(11, addr.ruteLordag);
                                        stmt.setString(12, addr.korelisteLordag);
                                        stmt.setString(13, addr.ruteSondag);
                                        stmt.setString(14, addr.korelisteSondag);

                                        stmt.setString(15, addr.distributor);
                                        stmt.setInt(16, addr.dbkBane);
                                        stmt.setInt(17, addr.id);
                                        
                                        
                                        stmt.addBatch();
                                        count++;
                                        if (count >= 100) {
                                                stmt.executeBatch();
                                                count = 0;
                                        }                                       
                                }
                                
                                stmt.executeBatch(); //update remainding queries
                                
                        }
                        
                        //con.commit();
                } catch (SQLException e) {
                        logger.warning("Error updating addresses", e );
                }
        }
        
        /*
        private static String nullify(String str) {
                if (str == null)
                        return null;
                
                if (str.equals("")) {
                        return null;
                } else {
                        return str;
                }
        }


        /*
        private static int safeInt(String str) {
                try {
                        return Integer.parseInt( str );
                } catch (NumberFormatException e) {
                        return 0;
                }
        }
        

        private static String coalesce(String s1, String s2) {
                if (s1 != null)
                        return s1;
                
                return s2;
        }

        */
}
1	torben	2838	package dk.daoas.adressevedligehold.db;
2
3			import java.sql.Connection;
4			import java.sql.PreparedStatement;
5			import java.sql.ResultSet;
6			import java.sql.SQLException;
7			import java.sql.Statement;
8			import java.util.ArrayList;
9			import java.util.List;
10
11	torben	2876
12	torben	2903
13	torben	2838	import dk.daoas.adressevedligehold.beans.Address;
14	torben	2876	import dk.daoas.adressevedligehold.beans.Address.AddressState;
15	torben	2903	import dk.daoas.adressevedligehold.tasks.TaskLogger;
16	torben	2838	import dk.daoas.adressevedligehold.util.DeduplicateHelper;
17			import dk.daoas.adressevedligehold.util.TimingHelper;
18
19	torben	2950	/*
20	torben	2840	*
21			*/
22	torben	2838
23	torben	2946	public class DatabaseCoverageUpdate {
24	torben	2838
25	torben	2847	//static boolean DEBUG = false;
26	torben	2838
27	torben	2903	private TaskLogger logger = TaskLogger.getInstance();
28	torben	2838
29	torben	2903
30	torben	2838	public List<Address> getAllAdresses() throws SQLException {
31			TimingHelper timing = new TimingHelper();
32
33	torben	2847	//String debugFilter = DatabaseLayerImplementation.DEBUG ? " WHERE postnr >= 6000 " : "";
34	torben	2838
35			String sql =
36			"SELECT id,vejnavn,husnr,husnrbogstav,kommunekode,vejkode,postnr,gadeid,upper(distributor) AS distributor,dbkbane,latitude,longitude, "
37			+ "rutema,korelistema,ruteti,korelisteti,ruteon,korelisteon,ruteto,korelisteto,rutefr,korelistefr,rutelo,korelistelo,ruteso,korelisteso "
38			+ "FROM fulddaekning.adressetabel "
39	torben	2847	//+ debugFilter
40	torben	2838	;
41
42			try ( Connection conn = DBConnection.getConnection();
43	torben	2937	Statement stmt = conn.createStatement(ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);
44	torben	2838	) {
45			stmt.setFetchSize(Integer.MIN_VALUE);
46			ResultSet res = stmt.executeQuery(sql);
47
48			List<Address> list = new ArrayList<Address>(2600000);//initial capacity 2.6 mio
49
50			DeduplicateHelper<String> vejnavnCache = new DeduplicateHelper<String>(81920);
51			DeduplicateHelper<String> husnrbogstavCache = new DeduplicateHelper<String>();
52			DeduplicateHelper<String> distributorCache = new DeduplicateHelper<String>();
53			DeduplicateHelper<String> dirigeringsCache = new DeduplicateHelper<String>(16*1024);
54
55
56			while (res.next()) {
57
58			Address a = new Address();
59			a.id = res.getInt(1);
60			a.vejnavn = vejnavnCache.getInstance( res.getString(2) );
61			a.husnr = (short) res.getInt(3);
62			a.husnrbogstav = husnrbogstavCache.getInstance( res.getString(4) );
63			a.kommunekode = (short) res.getInt(5);
64			a.vejkode = (short)res.getInt(6);
65			a.postnr = (short)res.getInt(7);
66			a.gadeid = res.getInt(8);
67			a.distributor = distributorCache.getInstance(res.getString(9));
68			a.dbkBane = (short) res.getInt(10);
69			a.latitude = (float) res.getDouble(11);
70			a.longitude = (float) res.getDouble(12);
71
72			a.ruteMandag = dirigeringsCache.getInstance( res.getString(13) );
73			a.korelisteMandag = dirigeringsCache.getInstance( res.getString(14) );
74
75			a.ruteTirsdag = dirigeringsCache.getInstance( res.getString(15) );
76			a.korelisteTirsdag = dirigeringsCache.getInstance( res.getString(16) );
77
78			a.ruteOnsdag = dirigeringsCache.getInstance( res.getString(17) );
79			a.korelisteOnsdag = dirigeringsCache.getInstance( res.getString(18) );
80
81			a.ruteTorsdag = dirigeringsCache.getInstance( res.getString(19) );
82			a.korelisteTorsdag = dirigeringsCache.getInstance( res.getString(20) );
83
84			a.ruteFredag = dirigeringsCache.getInstance( res.getString(21) );
85			a.korelisteFredag = dirigeringsCache.getInstance( res.getString(22) );
86
87			a.ruteLordag = dirigeringsCache.getInstance( res.getString(23) );
88			a.korelisteLordag = dirigeringsCache.getInstance( res.getString(24) );
89
90			a.ruteSondag = dirigeringsCache.getInstance( res.getString(25) );
91			a.korelisteSondag = dirigeringsCache.getInstance( res.getString(26) );
92
93
94			list.add(a);
95			}
96			res.close();
97
98
99	torben	2903	logger.info("DB Loaded " + list.size() + " adresses in " + timing.getElapsed() + "ms");
100	torben	2838
101			return list;
102			}
103			}
104
105	torben	2876	public void saveNewAddresses(List<Address> addresses) {
106	torben	2903	logger.info("DB::saveNewAddresses size:" + addresses.size() );
107	torben	2840
108	torben	2876	String sql = "INSERT IGNORE INTO fulddaekning.adressetabel " +
109			"(vejnavn,husnr,husnrbogstav,kommunekode,vejkode,postnr,gadeid,dbkbane,indlast,aendret,david,googlechecked," +
110	torben	3003	"RuteMa,KorelisteMa,RuteTi,KorelisteTi,RuteOn,KorelisteOn,RuteTo,KorelisteTo,RuteFr,KorelisteFr,RuteLo,KorelisteLo,RuteSo,KorelisteSo, " +
111			"kommentar,latitude,longitude) " +
112	torben	2876	"VALUES " +
113			"(?,?,?,?,?,?,?,?, now(), now(), ?, 0, " +
114	torben	3003	"?,?,?,?,?,?,?,?,?,?,?,?,?,?," +
115			"?,?,?) ";
116	torben	2876
117
118			try (Connection con = DBConnection.getConnection()) {
119	torben	2950
120	torben	2876
121			try (PreparedStatement stmt = con.prepareStatement(sql)) {
122			for (Address addr : addresses) {
123
124			String david = String.format("%04d%03d%04d", addr.postnr, addr.kommunekode, addr.vejkode);
125
126			stmt.setString(1, addr.vejnavn);
127			stmt.setShort(2, addr.husnr);
128			stmt.setString(3, addr.husnrbogstav);
129			stmt.setShort(4, addr.kommunekode);
130			stmt.setShort(5, addr.vejkode);
131			stmt.setShort(6, addr.postnr);
132			stmt.setInt(7, addr.gadeid);
133			stmt.setShort(8, addr.dbkBane);
134			stmt.setString(9, david);
135
136			stmt.setString(10, addr.ruteMandag);
137			stmt.setString(11, addr.korelisteMandag);
138			stmt.setString(12, addr.ruteTirsdag);
139			stmt.setString(13, addr.korelisteTirsdag);
140			stmt.setString(14, addr.ruteOnsdag);
141			stmt.setString(15, addr.korelisteOnsdag);
142			stmt.setString(16, addr.ruteTorsdag);
143			stmt.setString(17, addr.korelisteTorsdag);
144			stmt.setString(18, addr.ruteFredag);
145			stmt.setString(19, addr.korelisteFredag);
146			stmt.setString(20, addr.ruteLordag);
147			stmt.setString(21, addr.korelisteLordag);
148			stmt.setString(22, addr.ruteSondag);
149			stmt.setString(23, addr.korelisteSondag);
150	torben	3003
151			stmt.setString(24, addr.kommentar);
152
153			if (addr.latitude != 0.0) {
154			stmt.setDouble(25, addr.latitude);
155			} else {
156			stmt.setNull(25, java.sql.Types.DOUBLE);
157			}
158			if (addr.longitude != 0.0) {
159			stmt.setDouble(26, addr.longitude);
160			} else {
161			stmt.setNull(26, java.sql.Types.DOUBLE);
162			}
163	torben	2876
164
165			stmt.execute();
166			if (stmt.getUpdateCount() != 1) {
167	torben	2903	logger.info("Error executing insert: " + addr);
168	torben	2876	addr.state = AddressState.CREATE_FAIL;
169			}
170
171			}
172			}
173
174			//con.commit();
175			} catch (SQLException e) {
176	torben	2903	logger.warning("Error inserting addresses", e );
177	torben	2876	}
178	torben	2840	}
179
180	torben	2838	public void updateAddresses(List<Address> addresses) {
181	torben	2903	logger.info("DB::UpdateAddresses size:" + addresses.size() );
182	torben	2838
183			String sql = "UPDATE fulddaekning.adressetabel " +
184			"SET " +
185			"RuteMa=?, KorelisteMa=?, " +
186			"RuteTi=?, KorelisteTi=?, " +
187			"RuteOn=?, KorelisteOn=?, " +
188			"RuteTo=?, KorelisteTo=?, " +
189			"RuteFr=?, KorelisteFr=?, " +
190			"RuteLo=?, KorelisteLo=?, " +
191	torben	2845	"RuteSo=?, KorelisteSo=?, " +
192	torben	2847	"Distributor=?, " +
193	torben	2853	"DBKBane=?, " +
194			"Aendret=now() " +
195			"WHERE id=? ";
196	torben	2838
197
198
199			try (Connection con = DBConnection.getConnection()) {
200	torben	2845	//con.setAutoCommit(false);
201	torben	2838
202			try (PreparedStatement stmt = con.prepareStatement(sql)) {
203	torben	3018	int count = 0;
204
205	torben	2838	for (Address addr : addresses) {
206			stmt.setString(1, addr.ruteMandag);
207			stmt.setString(2, addr.korelisteMandag);
208			stmt.setString(3, addr.ruteTirsdag);
209			stmt.setString(4, addr.korelisteTirsdag);
210			stmt.setString(5, addr.ruteOnsdag);
211			stmt.setString(6, addr.korelisteOnsdag);
212			stmt.setString(7, addr.ruteTorsdag);
213			stmt.setString(8, addr.korelisteTorsdag);
214			stmt.setString(9, addr.ruteFredag);
215			stmt.setString(10, addr.korelisteFredag);
216			stmt.setString(11, addr.ruteLordag);
217			stmt.setString(12, addr.korelisteLordag);
218			stmt.setString(13, addr.ruteSondag);
219			stmt.setString(14, addr.korelisteSondag);
220
221	torben	2845	stmt.setString(15, addr.distributor);
222	torben	2847	stmt.setInt(16, addr.dbkBane);
223			stmt.setInt(17, addr.id);
224	torben	2838
225
226	torben	3018	stmt.addBatch();
227			count++;
228			if (count >= 100) {
229			stmt.executeBatch();
230			count = 0;
231			}
232	torben	2838	}
233	torben	3018
234			stmt.executeBatch(); //update remainding queries
235
236	torben	2838	}
237
238	torben	2845	//con.commit();
239	torben	2838	} catch (SQLException e) {
240	torben	2903	logger.warning("Error updating addresses", e );
241	torben	2838	}
242			}
243
244	torben	2898	/*
245	torben	2838	private static String nullify(String str) {
246			if (str == null)
247			return null;
248
249			if (str.equals("")) {
250			return null;
251			} else {
252			return str;
253			}
254			}
255
256
257			/*
258			private static int safeInt(String str) {
259			try {
260			return Integer.parseInt( str );
261			} catch (NumberFormatException e) {
262			return 0;
263			}
264			}
265
266
267
268			private static String coalesce(String s1, String s2) {
269			if (s1 != null)
270			return s1;
271
272			return s2;
273			}
274
275			*/
276			}