adressevedligehold/afstandandenrute/Database.java

package dk.daoas.adressevedligehold.afstandandenrute;

import java.io.IOException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentLinkedQueue;

import dk.daoas.adressevedligehold.beans.Address;
import dk.daoas.adressevedligehold.tasks.TaskLogger;
import dk.daoas.adressevedligehold.util.DeduplicateHelper;
import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;


public class Database { 
        private TaskLogger logger = TaskLogger.getInstance();

        int batchCount = 0;

        Connection conn;
        PreparedStatement saveStmt;
        
        List<Address> alleAddressr;
        Address alleIkkeDaekkede[];
        
        Map<Short,List<Address>> daekkedeAddressrHO = new HashMap<Short,List<Address>>();
        
        
        DeduplicateHelper<String> vejnavnCache = new DeduplicateHelper<String>();
        DeduplicateHelper<String> husnrbogstavCache = new DeduplicateHelper<String>();
        DeduplicateHelper<String> ruteCache = new DeduplicateHelper<String>(); 
        
        Set<Short> postnumre = new TreeSet<Short>();
        
        //Map<Short, List<Address>> ikkeDaekkedePrPost = new HashMap<Short, List<Address>>(); 
        
        
        private HashMap<Short,BoundingBox> bbCache = new HashMap<Short,BoundingBox>();

        public Database(Connection conn)  throws SQLException,IOException {
                this.conn = conn;

                String sql = "INSERT INTO fulddaekning.afstand_anden_rute_ny (orgId,orgPostnr, orgAdresse,orgGadeid,orgHusnr,orgHusnrBogstav,orgRute,id,postnr,adresse,gadeid,husnr,husnrbogstav,rute,afstand,`timestamp`) "+
                                "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )";

                saveStmt = conn.prepareStatement(sql);          

        }

        public void resetResultTable() throws SQLException {
                try (Statement stmt = conn.createStatement()) {
                        logger.info("Dropping old result table (if exists)");
                        String sql = "DROP TABLE IF EXISTS fulddaekning.afstand_anden_rute_ny";
                        stmt.executeUpdate(sql);
                        
                        logger.info("Create new result table");
                        sql = "CREATE TABLE fulddaekning.afstand_anden_rute_ny LIKE fulddaekning.afstand_anden_rute";
                        stmt.executeUpdate(sql);                
                        
                }
        }       
        
        @SuppressFBWarnings("SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE")
        public void renameResultTables() throws SQLException {
                if (AfstandAndenRuteTask.test_mode == true) {
                        throw new RuntimeException("Can not rename tables in test mode");
                }
                
                Constants consts = Constants.getInstance();
                
                try (Statement stmt = conn.createStatement()) {
                        
                        String ext = consts.getTableExtension();
                        
                        logger.info("Dropping old backup table (if exists)");
                        String sql = "DROP TABLE IF EXISTS fulddaekning.afstand_anden_rute_old" + ext;
                        stmt.executeUpdate(sql);
                        
                        logger.info("Rename tables");
                        String sql2 = "RENAME TABLE fulddaekning.afstand_anden_rute" + ext + " TO fulddaekning.afstand_anden_rute_old" + ext + ", fulddaekning.afstand_anden_rute_ny TO fulddaekning.afstand_anden_rute" + ext;

                        logger.info("Executing: " + sql2);
                        stmt.executeUpdate(sql2);                       
                }

                
        }
        
        public BoundingBox getBoundingbox(short postnr)  {
                BoundingBox bb = bbCache.get(postnr);
                return new BoundingBox(bb); //never return the original / cached object
        }
        
        
        public Set<Short> hentPostnumreCache() {
                return postnumre;
        }


        public Queue<Address> hentAlleIkkedaekkedeAdresser(int minPostnr, int maxPostnr)  throws SQLException {
                
                logger.info("Henter alle IKKE-daekkede Addressr");

                String sql = "SELECT id,a.postnr,vejnavn,gadeid,husnr,husnrbogstav,latitude,longitude,ruteMa,p.distributor as ho " + 
                                "FROM fulddaekning.adressetabel a " +
                                "LEFT JOIN bogleveringer.postnummerdistributor p on (a.postnr=p.postnr) " +
                                "WHERE ruteMa IS NULL " +  //Ingen dækning
                                "AND a.postnr BETWEEN ? AND ? " +
                                "AND latitude IS NOT NULL " +
                                "AND longitude IS NOT NULL " +
                                "AND gadeid IS NOT NULL " +
                                "AND (a.distributor IS NULL OR a.distributor<>'LUKKET') " +
                                "ORDER BY gadeid "
                                ;               
                
                if (AfstandAndenRuteTask.test_mode == true) {
                        sql = sql + " LIMIT 100 ";
                }
                
                try (PreparedStatement stmt = conn.prepareStatement(sql)) {
                        
                
                        stmt.setInt(1, minPostnr);
                        stmt.setInt(2, maxPostnr);
        
                        List<Address> list = hentAdresseListe( stmt );
                        alleIkkeDaekkede = list.toArray( new Address[ list.size() ] );
                        
                        logger.info("Analyserer ikke-daekkede Addressr");
                        
                        for (Address a : alleIkkeDaekkede) {
                                
                                
                                BoundingBox bbox;
                                
                                if (! postnumre.contains(a.postnr )) {
                                        postnumre.add( a.postnr );
                                        
                                        bbox = new BoundingBox(); 
                                        
                                        bbCache.put( a.postnr, bbox);                                                                                   
                                        
                                } else {
                                         bbox = bbCache.get( a.postnr);
                                }
                                
                                bbox.latitudeMax = Math.max(bbox.latitudeMax, a.latitude);
                                bbox.latitudeMin = Math.min(bbox.latitudeMin, a.latitude);
                                bbox.longitudeMax = Math.max(bbox.longitudeMax, a.longitude);
                                bbox.longitudeMin = Math.min(bbox.longitudeMin, a.longitude);
                                
                                
                        }
                        return new ConcurrentLinkedQueue<Address>( list );
                }
        }
        

        public Map<Short, List<Address>> getDaekkedeAdresserHO() {
                return daekkedeAddressrHO;
        }
        
        public int hentAlleDaekkedeAdresser(String distributor) throws SQLException {
                
                if ( alleAddressr == null ) {
                        String sql = "SELECT id,a.postnr,vejnavn,gadeid,husnr,husnrbogstav,latitude,longitude,ruteMa,p.distributor as ho " + 
                                        "FROM fulddaekning.adressetabel a " +
                                        "LEFT JOIN bogleveringer.postnummerdistributor p on (a.postnr=p.postnr) " +
                                        "WHERE ruteMa IS NOT NULL " + 
                                        "AND latitude IS NOT NULL " +
                                        "AND longitude IS NOT NULL " +
                                        "AND a.distributor = ? ";

                        if (AfstandAndenRuteTask.test_mode == true) {
                                sql = sql + " AND a.postnr BETWEEN 6000 and 7200 ";
                        }
        
                        // Forward only + concur_read_only + fetchsize tvinger driver til at hente en række af gangen (bedre performance ved store result sets)
                        // Se http://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html
                        
                        try (PreparedStatement stmt = conn.prepareStatement(sql, java.sql.ResultSet.TYPE_FORWARD_ONLY, java.sql.ResultSet.CONCUR_READ_ONLY)) {
                                stmt.setFetchSize(Integer.MIN_VALUE);
                
                                stmt.setString(1, distributor);
                
                                List<Address> list = hentAdresseListe( stmt );
                                
                                alleAddressr = new ArrayList<Address>();
                                alleAddressr.addAll(list);
                                
                                for(Address addr : list) {
                                        short ho = addr.ho;
                                        List<Address> hoListe = daekkedeAddressrHO.get(ho);
                                        if ( hoListe == null) {
                                                hoListe = new ArrayList<Address>();
                                                daekkedeAddressrHO.put(ho, hoListe);                                    
                                        }
                                        
                                        hoListe.add(addr);                                                              
                                        
                                }
                        }
                }
                
                return alleAddressr.size();
                
        }
        
        
        public synchronized void gemResultat(Address orgAddress, Address bedsteAddress, double bedsteAfstand) throws SQLException {
                /*String sql = "INSERT INTO fulddaekning.afstand_anden_rute_thn (orgId,orgPostnr, orgVejnavn,orgGadeid,orgHusnr,orgHusnrBogstav,orgLatitude,orgLongitude,orgRute,id,postnr,vejnavn,gadeid,husnr,husnrbogstav,latitude,longitude,rute,afstand,`timestamp`) "+
                                "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )";


                PreparedStatement saveStmt = conn.prepareStatement(sql);*/

                saveStmt.setInt(1, orgAddress.id);
                saveStmt.setShort(2, orgAddress.postnr);
                saveStmt.setString(3, orgAddress.vejnavn);
                saveStmt.setInt(4, orgAddress.gadeid);
                saveStmt.setShort(5, orgAddress.husnr);
                saveStmt.setString(6, orgAddress.husnrbogstav);
                saveStmt.setString(7, orgAddress.ruteMandag);


                saveStmt.setInt(8, bedsteAddress.id);
                saveStmt.setShort(9, bedsteAddress.postnr);
                saveStmt.setString(10, bedsteAddress.vejnavn);
                saveStmt.setInt(11, bedsteAddress.gadeid);
                saveStmt.setShort(12, bedsteAddress.husnr);
                saveStmt.setString(13, bedsteAddress.husnrbogstav);
                saveStmt.setString(14, bedsteAddress.ruteMandag);

                saveStmt.setDouble(15, bedsteAfstand);

                saveStmt.addBatch();
                batchCount++;
                if (batchCount >= 1000) {
                        saveStmt.executeBatch();
                        batchCount = 0;
                }
                //saveStmt.executeUpdate();     
                //saveStmt.clearParameters();

                //saveStmt.close();     
        }
        
        public synchronized void saveBatch() throws SQLException{
                saveStmt.executeBatch();
                batchCount = 0;
        }


        protected ArrayList<Address> hentAdresseListe(PreparedStatement stmt) throws SQLException{
                ArrayList<Address> list = new ArrayList<Address>( 1000000 );
                
                Constants consts = Constants.getInstance();

                //logger.info("Starting query");
                try ( ResultSet res = stmt.executeQuery() ) {
                

                        while (res.next()) {
                                
                                double latitude = res.getDouble(7);
                                double longitude = res.getDouble(8);
        
                                
                                Address adr = new Address(latitude,longitude);
        
                                adr.id = res.getInt(1);
                                adr.postnr = res.getShort(2);
                                adr.vejnavn = vejnavnCache.getInstance( res.getString(3) );
                                adr.gadeid = res.getInt(4);
                                adr.husnr = res.getShort(5);
                                adr.husnrbogstav = husnrbogstavCache.getInstance( res.getString(6) );
                                //adr.latitude = res.getDouble(7);
                                //adr.longitude = res.getDouble(8);
                                adr.ruteMandag =  ruteCache.getInstance( res.getString(9) );
                                adr.ho = res.getShort(10);
        
                                list.add(adr);
        
                                if (consts.doCheckHO() == true && adr.ho == 0) {
                                        logger.info( "Mangler HO: " + adr );
                                        throw new RuntimeException("Mangler HO: "+ adr);
                                }
        
                                //logger.info( "Adress:" + adr);
                        }
        
                        res.close();
                        stmt.close();
        
                        return list;
                }
        }
                
}
1	package dk.daoas.adressevedligehold.afstandandenrute;
2
3	import java.io.IOException;
4	import java.sql.Connection;
5	import java.sql.PreparedStatement;
6	import java.sql.ResultSet;
7	import java.sql.SQLException;
8	import java.sql.Statement;
9	import java.util.ArrayList;
10	import java.util.HashMap;
11	import java.util.List;
12	import java.util.Map;
13	import java.util.Queue;
14	import java.util.Set;
15	import java.util.TreeSet;
16	import java.util.concurrent.ConcurrentLinkedQueue;
17
18	import dk.daoas.adressevedligehold.beans.Address;
19	import dk.daoas.adressevedligehold.tasks.TaskLogger;
20	import dk.daoas.adressevedligehold.util.DeduplicateHelper;
21	import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
22
23
24
25	public class Database {
26	private TaskLogger logger = TaskLogger.getInstance();
27
28	int batchCount = 0;
29
30	Connection conn;
31	PreparedStatement saveStmt;
32
33	List<Address> alleAddressr;
34	Address alleIkkeDaekkede[];
35
36	Map<Short,List<Address>> daekkedeAddressrHO = new HashMap<Short,List<Address>>();
37
38
39
40
41
42	DeduplicateHelper<String> vejnavnCache = new DeduplicateHelper<String>();
43	DeduplicateHelper<String> husnrbogstavCache = new DeduplicateHelper<String>();
44	DeduplicateHelper<String> ruteCache = new DeduplicateHelper<String>();
45
46	Set<Short> postnumre = new TreeSet<Short>();
47
48	//Map<Short, List<Address>> ikkeDaekkedePrPost = new HashMap<Short, List<Address>>();
49
50
51	private HashMap<Short,BoundingBox> bbCache = new HashMap<Short,BoundingBox>();
52
53	public Database(Connection conn) throws SQLException,IOException {
54	this.conn = conn;
55
56	String sql = "INSERT INTO fulddaekning.afstand_anden_rute_ny (orgId,orgPostnr, orgAdresse,orgGadeid,orgHusnr,orgHusnrBogstav,orgRute,id,postnr,adresse,gadeid,husnr,husnrbogstav,rute,afstand,`timestamp`) "+
57	"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )";
58
59	saveStmt = conn.prepareStatement(sql);
60
61	}
62
63	public void resetResultTable() throws SQLException {
64	try (Statement stmt = conn.createStatement()) {
65	logger.info("Dropping old result table (if exists)");
66	String sql = "DROP TABLE IF EXISTS fulddaekning.afstand_anden_rute_ny";
67	stmt.executeUpdate(sql);
68
69	logger.info("Create new result table");
70	sql = "CREATE TABLE fulddaekning.afstand_anden_rute_ny LIKE fulddaekning.afstand_anden_rute";
71	stmt.executeUpdate(sql);
72
73	}
74	}
75
76	@SuppressFBWarnings("SQL_NONCONSTANT_STRING_PASSED_TO_EXECUTE")
77	public void renameResultTables() throws SQLException {
78	if (AfstandAndenRuteTask.test_mode == true) {
79	throw new RuntimeException("Can not rename tables in test mode");
80	}
81
82	Constants consts = Constants.getInstance();
83
84	try (Statement stmt = conn.createStatement()) {
85
86	String ext = consts.getTableExtension();
87
88	logger.info("Dropping old backup table (if exists)");
89	String sql = "DROP TABLE IF EXISTS fulddaekning.afstand_anden_rute_old" + ext;
90	stmt.executeUpdate(sql);
91
92	logger.info("Rename tables");
93	String sql2 = "RENAME TABLE fulddaekning.afstand_anden_rute" + ext + " TO fulddaekning.afstand_anden_rute_old" + ext + ", fulddaekning.afstand_anden_rute_ny TO fulddaekning.afstand_anden_rute" + ext;
94
95	logger.info("Executing: " + sql2);
96	stmt.executeUpdate(sql2);
97	}
98
99
100	}
101
102	public BoundingBox getBoundingbox(short postnr) {
103	BoundingBox bb = bbCache.get(postnr);
104	return new BoundingBox(bb); //never return the original / cached object
105	}
106
107
108	public Set<Short> hentPostnumreCache() {
109	return postnumre;
110	}
111
112
113	public Queue<Address> hentAlleIkkedaekkedeAdresser(int minPostnr, int maxPostnr) throws SQLException {
114
115	logger.info("Henter alle IKKE-daekkede Addressr");
116
117	String sql = "SELECT id,a.postnr,vejnavn,gadeid,husnr,husnrbogstav,latitude,longitude,ruteMa,p.distributor as ho " +
118	"FROM fulddaekning.adressetabel a " +
119	"LEFT JOIN bogleveringer.postnummerdistributor p on (a.postnr=p.postnr) " +
120	"WHERE ruteMa IS NULL " + //Ingen dækning
121	"AND a.postnr BETWEEN ? AND ? " +
122	"AND latitude IS NOT NULL " +
123	"AND longitude IS NOT NULL " +
124	"AND gadeid IS NOT NULL " +
125	"AND (a.distributor IS NULL OR a.distributor<>'LUKKET') " +
126	"ORDER BY gadeid "
127	;
128
129	if (AfstandAndenRuteTask.test_mode == true) {
130	sql = sql + " LIMIT 100 ";
131	}
132
133	try (PreparedStatement stmt = conn.prepareStatement(sql)) {
134
135
136	stmt.setInt(1, minPostnr);
137	stmt.setInt(2, maxPostnr);
138
139	List<Address> list = hentAdresseListe( stmt );
140	alleIkkeDaekkede = list.toArray( new Address[ list.size() ] );
141
142	logger.info("Analyserer ikke-daekkede Addressr");
143
144	for (Address a : alleIkkeDaekkede) {
145
146
147	BoundingBox bbox;
148
149	if (! postnumre.contains(a.postnr )) {
150	postnumre.add( a.postnr );
151
152	bbox = new BoundingBox();
153
154	bbCache.put( a.postnr, bbox);
155
156	} else {
157	bbox = bbCache.get( a.postnr);
158	}
159
160	bbox.latitudeMax = Math.max(bbox.latitudeMax, a.latitude);
161	bbox.latitudeMin = Math.min(bbox.latitudeMin, a.latitude);
162	bbox.longitudeMax = Math.max(bbox.longitudeMax, a.longitude);
163	bbox.longitudeMin = Math.min(bbox.longitudeMin, a.longitude);
164
165
166	}
167	return new ConcurrentLinkedQueue<Address>( list );
168	}
169	}
170
171
172
173
174	public Map<Short, List<Address>> getDaekkedeAdresserHO() {
175	return daekkedeAddressrHO;
176	}
177
178	public int hentAlleDaekkedeAdresser(String distributor) throws SQLException {
179
180	if ( alleAddressr == null ) {
181	String sql = "SELECT id,a.postnr,vejnavn,gadeid,husnr,husnrbogstav,latitude,longitude,ruteMa,p.distributor as ho " +
182	"FROM fulddaekning.adressetabel a " +
183	"LEFT JOIN bogleveringer.postnummerdistributor p on (a.postnr=p.postnr) " +
184	"WHERE ruteMa IS NOT NULL " +
185	"AND latitude IS NOT NULL " +
186	"AND longitude IS NOT NULL " +
187	"AND a.distributor = ? ";
188
189	if (AfstandAndenRuteTask.test_mode == true) {
190	sql = sql + " AND a.postnr BETWEEN 6000 and 7200 ";
191	}
192
193	// Forward only + concur_read_only + fetchsize tvinger driver til at hente en række af gangen (bedre performance ved store result sets)
194	// Se http://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html
195
196	try (PreparedStatement stmt = conn.prepareStatement(sql, java.sql.ResultSet.TYPE_FORWARD_ONLY, java.sql.ResultSet.CONCUR_READ_ONLY)) {
197	stmt.setFetchSize(Integer.MIN_VALUE);
198
199	stmt.setString(1, distributor);
200
201	List<Address> list = hentAdresseListe( stmt );
202
203	alleAddressr = new ArrayList<Address>();
204	alleAddressr.addAll(list);
205
206	for(Address addr : list) {
207	short ho = addr.ho;
208	List<Address> hoListe = daekkedeAddressrHO.get(ho);
209	if ( hoListe == null) {
210	hoListe = new ArrayList<Address>();
211	daekkedeAddressrHO.put(ho, hoListe);
212	}
213
214	hoListe.add(addr);
215
216	}
217	}
218	}
219
220	return alleAddressr.size();
221
222	}
223
224
225
226	public synchronized void gemResultat(Address orgAddress, Address bedsteAddress, double bedsteAfstand) throws SQLException {
227	/*String sql = "INSERT INTO fulddaekning.afstand_anden_rute_thn (orgId,orgPostnr, orgVejnavn,orgGadeid,orgHusnr,orgHusnrBogstav,orgLatitude,orgLongitude,orgRute,id,postnr,vejnavn,gadeid,husnr,husnrbogstav,latitude,longitude,rute,afstand,`timestamp`) "+
228	"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )";
229
230
231	PreparedStatement saveStmt = conn.prepareStatement(sql);*/
232
233	saveStmt.setInt(1, orgAddress.id);
234	saveStmt.setShort(2, orgAddress.postnr);
235	saveStmt.setString(3, orgAddress.vejnavn);
236	saveStmt.setInt(4, orgAddress.gadeid);
237	saveStmt.setShort(5, orgAddress.husnr);
238	saveStmt.setString(6, orgAddress.husnrbogstav);
239	saveStmt.setString(7, orgAddress.ruteMandag);
240
241
242	saveStmt.setInt(8, bedsteAddress.id);
243	saveStmt.setShort(9, bedsteAddress.postnr);
244	saveStmt.setString(10, bedsteAddress.vejnavn);
245	saveStmt.setInt(11, bedsteAddress.gadeid);
246	saveStmt.setShort(12, bedsteAddress.husnr);
247	saveStmt.setString(13, bedsteAddress.husnrbogstav);
248	saveStmt.setString(14, bedsteAddress.ruteMandag);
249
250	saveStmt.setDouble(15, bedsteAfstand);
251
252	saveStmt.addBatch();
253	batchCount++;
254	if (batchCount >= 1000) {
255	saveStmt.executeBatch();
256	batchCount = 0;
257	}
258	//saveStmt.executeUpdate();
259	//saveStmt.clearParameters();
260
261	//saveStmt.close();
262	}
263
264	public synchronized void saveBatch() throws SQLException{
265	saveStmt.executeBatch();
266	batchCount = 0;
267	}
268
269
270
271	protected ArrayList<Address> hentAdresseListe(PreparedStatement stmt) throws SQLException{
272	ArrayList<Address> list = new ArrayList<Address>( 1000000 );
273
274	Constants consts = Constants.getInstance();
275
276	//logger.info("Starting query");
277	try ( ResultSet res = stmt.executeQuery() ) {
278
279
280	while (res.next()) {
281
282	double latitude = res.getDouble(7);
283	double longitude = res.getDouble(8);
284
285
286	Address adr = new Address(latitude,longitude);
287
288	adr.id = res.getInt(1);
289	adr.postnr = res.getShort(2);
290	adr.vejnavn = vejnavnCache.getInstance( res.getString(3) );
291	adr.gadeid = res.getInt(4);
292	adr.husnr = res.getShort(5);
293	adr.husnrbogstav = husnrbogstavCache.getInstance( res.getString(6) );
294	//adr.latitude = res.getDouble(7);
295	//adr.longitude = res.getDouble(8);
296	adr.ruteMandag = ruteCache.getInstance( res.getString(9) );
297	adr.ho = res.getShort(10);
298
299	list.add(adr);
300
301	if (consts.doCheckHO() == true && adr.ho == 0) {
302	logger.info( "Mangler HO: " + adr );
303	throw new RuntimeException("Mangler HO: "+ adr);
304	}
305
306	//logger.info( "Adress:" + adr);
307	}
308
309	res.close();
310	stmt.close();
311
312	return list;
313	}
314	}
315
316	}