adressevedligehold/afstandandenrute/Database.java

package dk.daoas.adressevedligehold.afstandandenrute;

import java.io.IOException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Queue;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.logging.Logger;

import dk.daoas.adressevedligehold.beans.Address;
import dk.daoas.adressevedligehold.util.DeduplicateHelper;


public class Database { 
        Logger logger = Logger.getLogger(Database.class.getName());

        int batchCount = 0;

        Connection conn;
        PreparedStatement saveStmt;
        
        List<Address> alleAddressr;
        Address alleIkkeDaekkede[];
        
        Map<Short,List<Address>> daekkedeAddressrHO = new HashMap<Short,List<Address>>();
        
        
        DeduplicateHelper<String> vejnavnCache = new DeduplicateHelper<String>();
        DeduplicateHelper<String> husnrbogstavCache = new DeduplicateHelper<String>();
        DeduplicateHelper<String> ruteCache = new DeduplicateHelper<String>(); 
        
        Set<Short> postnumre = new TreeSet<Short>();
        
        //Map<Short, List<Address>> ikkeDaekkedePrPost = new HashMap<Short, List<Address>>(); 
        
        
        private HashMap<Short,BoundingBox> bbCache = new HashMap<Short,BoundingBox>();

        public Database(Connection conn)  throws SQLException,IOException {
                this.conn = conn;

                String sql = "INSERT INTO fulddaekning.afstand_anden_rute_ny (orgId,orgPostnr, orgAddress,orgGadeid,orgHusnr,orgHusnrBogstav,orgRute,id,postnr,Address,gadeid,husnr,husnrbogstav,rute,afstand,`timestamp`) "+
                                "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )";

                saveStmt = conn.prepareStatement(sql);          

        }

        public void resetResultTable() throws SQLException {
                logger.info("Dropping old result table (if exists)");
                String sql = "DROP TABLE IF EXISTS fulddaekning.afstand_anden_rute_ny";
                conn.createStatement().executeUpdate(sql);
                
                logger.info("Create new result table");
                sql = "CREATE TABLE fulddaekning.afstand_anden_rute_ny LIKE fulddaekning.afstand_anden_rute";
                conn.createStatement().executeUpdate(sql);              
        }       
        
        public void renameResultTables() throws SQLException {
                if (AfstandAndenRuteTask.test_mode == true) {
                        throw new RuntimeException("Can not rename tables in test mode");
                }

                Constants consts = Constants.getInstance();
                String ext = consts.getTableExtension();
                
                logger.info("Dropping old backup table (if exists)");
                String sql = "DROP TABLE IF EXISTS fulddaekning.afstand_anden_rute_old" + ext;
                conn.createStatement().executeUpdate(sql);
                
                logger.info("Rename tables");
                String sql2 = "RENAME TABLE fulddaekning.afstand_anden_rute" + ext + " TO fulddaekning.afstand_anden_rute_old" + ext + ", fulddaekning.afstand_anden_rute_ny TO fulddaekning.afstand_anden_rute" + ext;

                logger.info("Executing: " + sql2);
                conn.createStatement().executeUpdate(sql2);             
        }
        
        public BoundingBox getBoundingbox(short postnr)  {
                BoundingBox bb = bbCache.get(postnr);
                return bb.clone();//never return the original / cached object
        }
        
        
        public Set<Short> hentPostnumreCache() {
                return postnumre;
        }


        public Queue<Address> hentAlleIkkedaekkedeAdresser(int minPostnr, int maxPostnr)  throws SQLException {
                
                logger.info("Henter alle IKKE-daekkede Addressr");

                String sql = "SELECT id,a.postnr,vejnavn,gadeid,husnr,husnrbogstav,latitude,longitude,ruteMa,p.distributor as ho " + 
                                "FROM fulddaekning.Addresstabel a " +
                                "LEFT JOIN bogleveringer.postnummerdistributor p on (a.postnr=p.postnr) " +
                                "WHERE ruteMa IS NULL " +  //Ingen dækning
                                "AND a.postnr BETWEEN ? AND ? " +
                                "AND latitude IS NOT NULL " +
                                "AND longitude IS NOT NULL " +
                                "AND gadeid IS NOT NULL " +
                                "AND (a.distributor IS NULL OR a.distributor<>'LUKKET') " 
                                ;               
                
                if (AfstandAndenRuteTask.test_mode == true) {
                        sql = sql + " LIMIT 100 ";
                }
                
                PreparedStatement stmt = conn.prepareStatement(sql);
                stmt.setInt(1, minPostnr);
                stmt.setInt(2, maxPostnr);

                List<Address> list = hentAdresseListe( stmt );
                alleIkkeDaekkede = list.toArray( new Address[ list.size() ] );
                
                logger.info("Analyserer ikke-daekkede Addressr");
                
                for (Address a : alleIkkeDaekkede) {
                        
                        
                        BoundingBox bbox;
                        
                        if (! postnumre.contains(a.postnr )) {
                                postnumre.add( a.postnr );
                                
                                bbox = new BoundingBox(); 
                                
                                bbCache.put( a.postnr, bbox);                                                                                   
                                
                        } else {
                                 bbox = bbCache.get( a.postnr);
                        }
                        
                        bbox.latitudeMax = Math.max(bbox.latitudeMax, a.latitude);
                        bbox.latitudeMin = Math.min(bbox.latitudeMin, a.latitude);
                        bbox.longitudeMax = Math.max(bbox.longitudeMax, a.longitude);
                        bbox.longitudeMin = Math.min(bbox.longitudeMin, a.longitude);
                        
                        
                }
                return new ConcurrentLinkedQueue<Address>( list );
        }
        

        public Map<Short, List<Address>> getDaekkedeAdresserHO() {
                return daekkedeAddressrHO;
        }
        
        public int hentAlleDaekkedeAdresser(String distributor) throws SQLException {
                
                if ( alleAddressr == null ) {
                        String sql = "SELECT id,a.postnr,vejnavn,gadeid,husnr,husnrbogstav,latitude,longitude,ruteMa,p.distributor as ho " + 
                                        "FROM fulddaekning.Addresstabel a " +
                                        "LEFT JOIN bogleveringer.postnummerdistributor p on (a.postnr=p.postnr) " +
                                        "WHERE ruteMa IS NOT NULL " + 
                                        "AND latitude IS NOT NULL " +
                                        "AND longitude IS NOT NULL " +
                                        "AND a.distributor = ? ";

                        if (AfstandAndenRuteTask.test_mode == true) {
                                sql = sql + " AND a.postnr BETWEEN 6000 and 7200 ";
                        }
        
                        // Forward only + concur_read_only + fetchsize tvinger driver til at hente en række af gangen (bedre performance ved store result sets)
                        // Se http://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html 
                        PreparedStatement stmt = conn.prepareStatement(sql, java.sql.ResultSet.TYPE_FORWARD_ONLY, java.sql.ResultSet.CONCUR_READ_ONLY);
                        stmt.setFetchSize(Integer.MIN_VALUE);
        
                        stmt.setString(1, distributor);
        
                        List<Address> list = hentAdresseListe( stmt );
                        
                        alleAddressr = new ArrayList<Address>();
                        alleAddressr.addAll(list);
                        
                        for(Address addr : list) {
                                short ho = addr.ho;
                                List<Address> hoListe = daekkedeAddressrHO.get(ho);
                                if ( hoListe == null) {
                                        hoListe = new ArrayList<Address>();
                                        daekkedeAddressrHO.put(ho, hoListe);                                    
                                }
                                
                                hoListe.add(addr);                                                              
                                
                        }
                }
                
                return alleAddressr.size();
        }
        
        
        public synchronized void gemResultat(Address orgAddress, Address bedsteAddress, double bedsteAfstand) throws SQLException {
                /*String sql = "INSERT INTO fulddaekning.afstand_anden_rute_thn (orgId,orgPostnr, orgVejnavn,orgGadeid,orgHusnr,orgHusnrBogstav,orgLatitude,orgLongitude,orgRute,id,postnr,vejnavn,gadeid,husnr,husnrbogstav,latitude,longitude,rute,afstand,`timestamp`) "+
                                "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )";


                PreparedStatement saveStmt = conn.prepareStatement(sql);*/

                saveStmt.setInt(1, orgAddress.id);
                saveStmt.setShort(2, orgAddress.postnr);
                saveStmt.setString(3, orgAddress.vejnavn);
                saveStmt.setInt(4, orgAddress.gadeid);
                saveStmt.setShort(5, orgAddress.husnr);
                saveStmt.setString(6, orgAddress.husnrbogstav);
                saveStmt.setString(7, orgAddress.ruteMandag);


                saveStmt.setInt(8, bedsteAddress.id);
                saveStmt.setShort(9, bedsteAddress.postnr);
                saveStmt.setString(10, bedsteAddress.vejnavn);
                saveStmt.setInt(11, bedsteAddress.gadeid);
                saveStmt.setShort(12, bedsteAddress.husnr);
                saveStmt.setString(13, bedsteAddress.husnrbogstav);
                saveStmt.setString(14, bedsteAddress.ruteMandag);

                saveStmt.setDouble(15, bedsteAfstand);

                saveStmt.addBatch();
                batchCount++;
                if (batchCount >= 1000) {
                        saveStmt.executeBatch();
                        batchCount = 0;
                }
                //saveStmt.executeUpdate();     
                //saveStmt.clearParameters();

                //saveStmt.close();     
        }
        
        public synchronized void saveBatch() throws SQLException{
                saveStmt.executeBatch();
                batchCount = 0;
        }


        protected ArrayList<Address> hentAdresseListe(PreparedStatement stmt) throws SQLException{
                ArrayList<Address> list = new ArrayList<Address>( 1000000 );
                
                Constants consts = Constants.getInstance();

                //logger.info("Starting query");
                ResultSet res = stmt.executeQuery();
                //logger.info("Starting exec query done");

                while (res.next()) {
                        
                        double latitude = res.getDouble(7);
                        double longitude = res.getDouble(8);

                        
                        Address adr = new Address(latitude,longitude);

                        adr.id = res.getInt(1);
                        adr.postnr = res.getShort(2);
                        adr.vejnavn = vejnavnCache.getInstance( res.getString(3) );
                        adr.gadeid = res.getInt(4);
                        adr.husnr = res.getShort(5);
                        adr.husnrbogstav = husnrbogstavCache.getInstance( res.getString(6) );
                        //adr.latitude = res.getDouble(7);
                        //adr.longitude = res.getDouble(8);
                        adr.ruteMandag =  ruteCache.getInstance( res.getString(9) );
                        adr.ho = res.getShort(10);

                        list.add(adr);

                        if (consts.doCheckHO() == true && adr.ho == 0) {
                                System.out.println( "Mangler HO" );
                                System.out.println( adr );
                                System.exit(0);
                        }

                        //logger.info( "Adress:" + adr);
                }

                res.close();
                stmt.close();

                return list;
        }
                
}
1	torben	2878	package dk.daoas.adressevedligehold.afstandandenrute;
2
3			import java.io.IOException;
4			import java.sql.Connection;
5			import java.sql.PreparedStatement;
6			import java.sql.ResultSet;
7			import java.sql.SQLException;
8			import java.util.ArrayList;
9			import java.util.HashMap;
10			import java.util.List;
11			import java.util.Map;
12			import java.util.Queue;
13			import java.util.Set;
14			import java.util.TreeSet;
15			import java.util.concurrent.ConcurrentLinkedQueue;
16			import java.util.logging.Logger;
17
18			import dk.daoas.adressevedligehold.beans.Address;
19			import dk.daoas.adressevedligehold.util.DeduplicateHelper;
20
21
22
23			public class Database {
24			Logger logger = Logger.getLogger(Database.class.getName());
25
26			int batchCount = 0;
27
28			Connection conn;
29			PreparedStatement saveStmt;
30
31			List<Address> alleAddressr;
32			Address alleIkkeDaekkede[];
33
34			Map<Short,List<Address>> daekkedeAddressrHO = new HashMap<Short,List<Address>>();
35
36
37
38
39
40			DeduplicateHelper<String> vejnavnCache = new DeduplicateHelper<String>();
41			DeduplicateHelper<String> husnrbogstavCache = new DeduplicateHelper<String>();
42			DeduplicateHelper<String> ruteCache = new DeduplicateHelper<String>();
43
44			Set<Short> postnumre = new TreeSet<Short>();
45
46			//Map<Short, List<Address>> ikkeDaekkedePrPost = new HashMap<Short, List<Address>>();
47
48
49			private HashMap<Short,BoundingBox> bbCache = new HashMap<Short,BoundingBox>();
50
51			public Database(Connection conn) throws SQLException,IOException {
52			this.conn = conn;
53
54			String sql = "INSERT INTO fulddaekning.afstand_anden_rute_ny (orgId,orgPostnr, orgAddress,orgGadeid,orgHusnr,orgHusnrBogstav,orgRute,id,postnr,Address,gadeid,husnr,husnrbogstav,rute,afstand,`timestamp`) "+
55			"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )";
56
57			saveStmt = conn.prepareStatement(sql);
58
59			}
60
61			public void resetResultTable() throws SQLException {
62			logger.info("Dropping old result table (if exists)");
63			String sql = "DROP TABLE IF EXISTS fulddaekning.afstand_anden_rute_ny";
64			conn.createStatement().executeUpdate(sql);
65
66			logger.info("Create new result table");
67			sql = "CREATE TABLE fulddaekning.afstand_anden_rute_ny LIKE fulddaekning.afstand_anden_rute";
68			conn.createStatement().executeUpdate(sql);
69			}
70
71			public void renameResultTables() throws SQLException {
72			if (AfstandAndenRuteTask.test_mode == true) {
73			throw new RuntimeException("Can not rename tables in test mode");
74			}
75
76			Constants consts = Constants.getInstance();
77			String ext = consts.getTableExtension();
78
79			logger.info("Dropping old backup table (if exists)");
80			String sql = "DROP TABLE IF EXISTS fulddaekning.afstand_anden_rute_old" + ext;
81			conn.createStatement().executeUpdate(sql);
82
83			logger.info("Rename tables");
84			String sql2 = "RENAME TABLE fulddaekning.afstand_anden_rute" + ext + " TO fulddaekning.afstand_anden_rute_old" + ext + ", fulddaekning.afstand_anden_rute_ny TO fulddaekning.afstand_anden_rute" + ext;
85
86			logger.info("Executing: " + sql2);
87			conn.createStatement().executeUpdate(sql2);
88			}
89
90			public BoundingBox getBoundingbox(short postnr) {
91			BoundingBox bb = bbCache.get(postnr);
92			return bb.clone();//never return the original / cached object
93			}
94
95
96			public Set<Short> hentPostnumreCache() {
97			return postnumre;
98			}
99
100
101			public Queue<Address> hentAlleIkkedaekkedeAdresser(int minPostnr, int maxPostnr) throws SQLException {
102
103			logger.info("Henter alle IKKE-daekkede Addressr");
104
105			String sql = "SELECT id,a.postnr,vejnavn,gadeid,husnr,husnrbogstav,latitude,longitude,ruteMa,p.distributor as ho " +
106			"FROM fulddaekning.Addresstabel a " +
107			"LEFT JOIN bogleveringer.postnummerdistributor p on (a.postnr=p.postnr) " +
108			"WHERE ruteMa IS NULL " + //Ingen dækning
109			"AND a.postnr BETWEEN ? AND ? " +
110			"AND latitude IS NOT NULL " +
111			"AND longitude IS NOT NULL " +
112			"AND gadeid IS NOT NULL " +
113			"AND (a.distributor IS NULL OR a.distributor<>'LUKKET') "
114			;
115
116			if (AfstandAndenRuteTask.test_mode == true) {
117			sql = sql + " LIMIT 100 ";
118			}
119
120			PreparedStatement stmt = conn.prepareStatement(sql);
121			stmt.setInt(1, minPostnr);
122			stmt.setInt(2, maxPostnr);
123
124			List<Address> list = hentAdresseListe( stmt );
125			alleIkkeDaekkede = list.toArray( new Address[ list.size() ] );
126
127			logger.info("Analyserer ikke-daekkede Addressr");
128
129			for (Address a : alleIkkeDaekkede) {
130
131
132			BoundingBox bbox;
133
134			if (! postnumre.contains(a.postnr )) {
135			postnumre.add( a.postnr );
136
137			bbox = new BoundingBox();
138
139			bbCache.put( a.postnr, bbox);
140
141			} else {
142			bbox = bbCache.get( a.postnr);
143			}
144
145			bbox.latitudeMax = Math.max(bbox.latitudeMax, a.latitude);
146			bbox.latitudeMin = Math.min(bbox.latitudeMin, a.latitude);
147			bbox.longitudeMax = Math.max(bbox.longitudeMax, a.longitude);
148			bbox.longitudeMin = Math.min(bbox.longitudeMin, a.longitude);
149
150
151			}
152			return new ConcurrentLinkedQueue<Address>( list );
153			}
154
155
156
157
158			public Map<Short, List<Address>> getDaekkedeAdresserHO() {
159			return daekkedeAddressrHO;
160			}
161
162			public int hentAlleDaekkedeAdresser(String distributor) throws SQLException {
163
164			if ( alleAddressr == null ) {
165			String sql = "SELECT id,a.postnr,vejnavn,gadeid,husnr,husnrbogstav,latitude,longitude,ruteMa,p.distributor as ho " +
166			"FROM fulddaekning.Addresstabel a " +
167			"LEFT JOIN bogleveringer.postnummerdistributor p on (a.postnr=p.postnr) " +
168			"WHERE ruteMa IS NOT NULL " +
169			"AND latitude IS NOT NULL " +
170			"AND longitude IS NOT NULL " +
171			"AND a.distributor = ? ";
172
173			if (AfstandAndenRuteTask.test_mode == true) {
174			sql = sql + " AND a.postnr BETWEEN 6000 and 7200 ";
175			}
176
177			// Forward only + concur_read_only + fetchsize tvinger driver til at hente en række af gangen (bedre performance ved store result sets)
178			// Se http://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html
179			PreparedStatement stmt = conn.prepareStatement(sql, java.sql.ResultSet.TYPE_FORWARD_ONLY, java.sql.ResultSet.CONCUR_READ_ONLY);
180			stmt.setFetchSize(Integer.MIN_VALUE);
181
182			stmt.setString(1, distributor);
183
184			List<Address> list = hentAdresseListe( stmt );
185
186			alleAddressr = new ArrayList<Address>();
187			alleAddressr.addAll(list);
188
189			for(Address addr : list) {
190			short ho = addr.ho;
191			List<Address> hoListe = daekkedeAddressrHO.get(ho);
192			if ( hoListe == null) {
193			hoListe = new ArrayList<Address>();
194			daekkedeAddressrHO.put(ho, hoListe);
195			}
196
197			hoListe.add(addr);
198
199			}
200			}
201
202			return alleAddressr.size();
203			}
204
205
206
207			public synchronized void gemResultat(Address orgAddress, Address bedsteAddress, double bedsteAfstand) throws SQLException {
208			/*String sql = "INSERT INTO fulddaekning.afstand_anden_rute_thn (orgId,orgPostnr, orgVejnavn,orgGadeid,orgHusnr,orgHusnrBogstav,orgLatitude,orgLongitude,orgRute,id,postnr,vejnavn,gadeid,husnr,husnrbogstav,latitude,longitude,rute,afstand,`timestamp`) "+
209			"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )";
210
211
212			PreparedStatement saveStmt = conn.prepareStatement(sql);*/
213
214			saveStmt.setInt(1, orgAddress.id);
215			saveStmt.setShort(2, orgAddress.postnr);
216			saveStmt.setString(3, orgAddress.vejnavn);
217			saveStmt.setInt(4, orgAddress.gadeid);
218			saveStmt.setShort(5, orgAddress.husnr);
219			saveStmt.setString(6, orgAddress.husnrbogstav);
220			saveStmt.setString(7, orgAddress.ruteMandag);
221
222
223			saveStmt.setInt(8, bedsteAddress.id);
224			saveStmt.setShort(9, bedsteAddress.postnr);
225			saveStmt.setString(10, bedsteAddress.vejnavn);
226			saveStmt.setInt(11, bedsteAddress.gadeid);
227			saveStmt.setShort(12, bedsteAddress.husnr);
228			saveStmt.setString(13, bedsteAddress.husnrbogstav);
229			saveStmt.setString(14, bedsteAddress.ruteMandag);
230
231			saveStmt.setDouble(15, bedsteAfstand);
232
233			saveStmt.addBatch();
234			batchCount++;
235			if (batchCount >= 1000) {
236			saveStmt.executeBatch();
237			batchCount = 0;
238			}
239			//saveStmt.executeUpdate();
240			//saveStmt.clearParameters();
241
242			//saveStmt.close();
243			}
244
245			public synchronized void saveBatch() throws SQLException{
246			saveStmt.executeBatch();
247			batchCount = 0;
248			}
249
250
251
252			protected ArrayList<Address> hentAdresseListe(PreparedStatement stmt) throws SQLException{
253			ArrayList<Address> list = new ArrayList<Address>( 1000000 );
254
255			Constants consts = Constants.getInstance();
256
257			//logger.info("Starting query");
258			ResultSet res = stmt.executeQuery();
259			//logger.info("Starting exec query done");
260
261			while (res.next()) {
262
263			double latitude = res.getDouble(7);
264			double longitude = res.getDouble(8);
265
266
267			Address adr = new Address(latitude,longitude);
268
269			adr.id = res.getInt(1);
270			adr.postnr = res.getShort(2);
271			adr.vejnavn = vejnavnCache.getInstance( res.getString(3) );
272			adr.gadeid = res.getInt(4);
273			adr.husnr = res.getShort(5);
274			adr.husnrbogstav = husnrbogstavCache.getInstance( res.getString(6) );
275			//adr.latitude = res.getDouble(7);
276			//adr.longitude = res.getDouble(8);
277			adr.ruteMandag = ruteCache.getInstance( res.getString(9) );
278			adr.ho = res.getShort(10);
279
280			list.add(adr);
281
282			if (consts.doCheckHO() == true && adr.ho == 0) {
283			System.out.println( "Mangler HO" );
284			System.out.println( adr );
285			System.exit(0);
286			}
287
288			//logger.info( "Adress:" + adr);
289			}
290
291			res.close();
292			stmt.close();
293
294			return list;
295			}
296
297			}