daoas/fulddaekning/Database.java

package dk.daoas.fulddaekning;

import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Queue;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.logging.Logger;


public class Database { 
        Logger logger = Logger.getLogger(Database.class.getName());

        int batchCount = 0;

        Connection conn;
        PreparedStatement saveStmt;
        
        List<Adresse> alleAdresser;
        Adresse alleIkkeDaekkede[];
        
        Map<Short,List<Adresse>> daekkedeAdresserHO = new HashMap<Short,List<Adresse>>();
        
        
        DeduplicateHelper<String> husnrbogstavCache = new DeduplicateHelper<String>();
        DeduplicateHelper<String> ruteCache = new DeduplicateHelper<String>(); 
        
        Set<Short> postnumre = new TreeSet<Short>();
        
        //Map<Short, List<Adresse>> ikkeDaekkedePrPost = new HashMap<Short, List<Adresse>>(); 
        
        
        private HashMap<Short,BoundingBox> bbCache = new HashMap<Short,BoundingBox>();

        public Database(SafeProperties conf)  throws SQLException,IOException {
                this.conn = getConnection( conf );      

                String sql = "INSERT INTO fulddaekning.afstand_anden_rute_ny (orgId,orgPostnr, orgAdresse,orgGadeid,orgHusnr,orgHusnrBogstav,orgRute,id,postnr,adresse,gadeid,husnr,husnrbogstav,rute,afstand,`timestamp`) "+
                                "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )";

                saveStmt = conn.prepareStatement(sql);          

        }

        public void resetResultTable() throws SQLException {
                logger.info("Dropping old result table (if exists)");
                String sql = "DROP TABLE IF EXISTS fulddaekning.afstand_anden_rute_ny";
                conn.createStatement().executeUpdate(sql);
                
                logger.info("Create new result table");
                sql = "CREATE TABLE fulddaekning.afstand_anden_rute_ny LIKE fulddaekning.afstand_anden_rute";
                conn.createStatement().executeUpdate(sql);              
        }       
        
        public void renameResultTables() throws SQLException {
                Constants consts = Constants.getInstance();
                String ext = consts.getTableExtension();
                
                logger.info("Dropping old backup table (if exists)");
                String sql = "DROP TABLE IF EXISTS fulddaekning.afstand_anden_rute_old" + ext;
                conn.createStatement().executeUpdate(sql);
                
                logger.info("Rename tables");
                String sql2 = "RENAME TABLE fulddaekning.afstand_anden_rute" + ext + " TO fulddaekning.afstand_anden_rute_old" + ext + ", fulddaekning.afstand_anden_rute_ny TO fulddaekning.afstand_anden_rute" + ext;

                logger.info("Executing: " + sql2);
                conn.createStatement().executeUpdate(sql2);             
        }
        
        public BoundingBox getBoundingbox(short postnr)  {
                BoundingBox bb = bbCache.get(postnr);
                return bb.clone();//never return the original / cached object
        }
        
        
        public Set<Short> hentPostnumreCache() {
                return postnumre;
        }


        public Queue<Adresse> hentAlleIkkedaekkedeAdresser(int minPostnr, int maxPostnr)  throws SQLException {
                
                logger.info("Henter alle IKKE-daekkede adresser");

                String sql = "SELECT id,a.postnr,adresse,gadeid,husnr,husnrbogstav,latitude,longitude,rute,p.distributor as ho " + 
                                "FROM fulddaekning.adressetabel a " +
                                "LEFT JOIN bogleveringer.postnummerdistributor p on (a.postnr=p.postnr) " +
                                "WHERE rute IS NULL " +  //Ingen dækning
                                //"AND a.postnr BETWEEN ? AND ? " +
                                "AND a.postnr BETWEEN 8000 and 8600 " + 
                                //"AND a.postnr BETWEEN 9000 AND 9300 " + 
                                "AND latitude IS NOT NULL " +
                                "AND longitude IS NOT NULL " +
                                "AND gadeid IS NOT NULL " +
                                "AND (a.distributor IS NULL OR a.distributor<>'LUKKET') " +
                                "LIMIT 2500";           
                PreparedStatement stmt = conn.prepareStatement(sql);
                //stmt.setInt(1, minPostnr);
                //stmt.setInt(2, maxPostnr);

                List<Adresse> list = hentAdresseListe( stmt );
                alleIkkeDaekkede = list.toArray( new Adresse[ list.size() ] );
                
                logger.info("Analyserer ikke-daekkede adresser");
                
                for (Adresse a : alleIkkeDaekkede) {
                        
                        
                        BoundingBox bbox;
                        
                        if (! postnumre.contains(a.postnr )) {
                                postnumre.add( a.postnr );
                                
                                bbox = new BoundingBox(); 
                                
                                bbCache.put( a.postnr, bbox);                                                                                   
                                
                        } else {
                                 bbox = bbCache.get( a.postnr);
                        }
                        
                        bbox.latitudeMax = Math.max(bbox.latitudeMax, a.latitude);
                        bbox.latitudeMin = Math.min(bbox.latitudeMin, a.latitude);
                        bbox.longitudeMax = Math.max(bbox.longitudeMax, a.longitude);
                        bbox.longitudeMin = Math.min(bbox.longitudeMin, a.longitude);
                        
                        
                }
                return new ConcurrentLinkedQueue<Adresse>( list );
        }
        

        public Map<Short, List<Adresse>> getDaekkedeAdresserHO() {
                return daekkedeAdresserHO;
        }
        
        public int hentAlleDaekkedeAdresser() throws SQLException {
                if ( alleAdresser == null ) {
                        String sql = "SELECT id,a.postnr,adresse,gadeid,husnr,husnrbogstav,latitude,longitude,rute,p.distributor as ho " + 
                                        "FROM fulddaekning.adressetabel a " +
                                        "LEFT JOIN bogleveringer.postnummerdistributor p on (a.postnr=p.postnr) " +
                                        "WHERE rute IS NOT NULL " + 
                                        "AND latitude IS NOT NULL " +
                                        "AND longitude IS NOT NULL " +
                                        "AND a.distributor = ? ";
        
                        // Forward only + concur_read_only + fetchsize tvinger driver til at hente en række af gangen (bedre performance ved store result sets)
                        // Se http://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html 
                        PreparedStatement stmt = conn.prepareStatement(sql, java.sql.ResultSet.TYPE_FORWARD_ONLY, java.sql.ResultSet.CONCUR_READ_ONLY);
                        stmt.setFetchSize(Integer.MIN_VALUE);
        
                        stmt.setString(1, LookupMain.distributor);
        
                        List<Adresse> list = hentAdresseListe( stmt );
                        
                        alleAdresser = new ArrayList<Adresse>();
                        alleAdresser.addAll(list);
                        
                        for(Adresse addr : list) {
                                short ho = addr.ho;
                                List<Adresse> hoListe = daekkedeAdresserHO.get(ho);
                                if ( hoListe == null) {
                                        hoListe = new ArrayList<Adresse>();
                                        daekkedeAdresserHO.put(ho, hoListe);                                    
                                }
                                
                                hoListe.add(addr);                                                              
                                
                        }
                }
                
                return alleAdresser.size();
        }
        
        
        public synchronized void gemResultat(Adresse orgAdresse, Adresse bedsteAdresse, double bedsteAfstand) throws SQLException {
                /*String sql = "INSERT INTO fulddaekning.afstand_anden_rute_thn (orgId,orgPostnr, orgAdresse,orgGadeid,orgHusnr,orgHusnrBogstav,orgLatitude,orgLongitude,orgRute,id,postnr,adresse,gadeid,husnr,husnrbogstav,latitude,longitude,rute,afstand,`timestamp`) "+
                                "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )";

                PreparedStatement saveStmt = conn.prepareStatement(sql);*/

                saveStmt.setInt(1, orgAdresse.id);
                saveStmt.setShort(2, orgAdresse.postnr);
                saveStmt.setString(3, orgAdresse.adresse);
                saveStmt.setInt(4, orgAdresse.gadeid);
                saveStmt.setShort(5, orgAdresse.husnr);
                saveStmt.setString(6, orgAdresse.husnrbogstav);
                saveStmt.setString(7, orgAdresse.rute);


                saveStmt.setInt(8, bedsteAdresse.id);
                saveStmt.setShort(9, bedsteAdresse.postnr);
                saveStmt.setString(10, bedsteAdresse.adresse);
                saveStmt.setInt(11, bedsteAdresse.gadeid);
                saveStmt.setShort(12, bedsteAdresse.husnr);
                saveStmt.setString(13, bedsteAdresse.husnrbogstav);
                saveStmt.setString(14, bedsteAdresse.rute);

                saveStmt.setDouble(15, bedsteAfstand);

                saveStmt.addBatch();
                batchCount++;
                if (batchCount >= 1000) {
                        saveStmt.executeBatch();
                        batchCount = 0;
                }
                //saveStmt.executeUpdate();     
                //saveStmt.clearParameters();

                //saveStmt.close();     
        }
        
        public synchronized void saveBatch() throws SQLException{
                saveStmt.executeBatch();
                batchCount = 0;
        }


        protected ArrayList<Adresse> hentAdresseListe(PreparedStatement stmt) throws SQLException{
                ArrayList<Adresse> list = new ArrayList<Adresse>( 1000000 );

                //logger.info("Starting query");
                ResultSet res = stmt.executeQuery();
                //logger.info("Starting exec query done");

                while (res.next()) {
                        
                        double latitude = res.getDouble(7);
                        double longitude = res.getDouble(8);

                        
                        Adresse adr = new Adresse(latitude,longitude);

                        adr.id = res.getInt(1);
                        adr.postnr = res.getShort(2);
                        adr.adresse = res.getString(3);
                        adr.gadeid = res.getInt(4);
                        adr.husnr = res.getShort(5);
                        adr.husnrbogstav = husnrbogstavCache.getInstance( res.getString(6) );
                        //adr.latitude = res.getDouble(7);
                        //adr.longitude = res.getDouble(8);
                        adr.rute =  ruteCache.getInstance( res.getString(9) );
                        adr.ho = res.getShort(10);

                        list.add(adr);

                        //logger.info( "Adress:" + adr);
                }

                res.close();
                stmt.close();

                return list;
        }

        public Connection getConnection(SafeProperties conf) throws SQLException, IOException {
                
                String db_host = conf.getSafeProperty("DB_HOST");
                String db_user = conf.getSafeProperty("DB_USER");
                String db_pass = conf.getSafeProperty("DB_PASS");

                
            Connection conn = null;
            Properties connectionProps = new Properties();
            connectionProps.put("user", db_user);
            connectionProps.put("password", db_pass);

            //For debug output, tilføj denne til JDBC url'en: &profileSQL=true     
            conn = DriverManager.getConnection(
                           "jdbc:mysql://" +
                           db_host +
                           ":3306/?rewriteBatchedStatements=true",
                           connectionProps);
            logger.info("Connected to database");
            return conn;
        }

                
}
1	package dk.daoas.fulddaekning;
2
3	import java.io.IOException;
4	import java.sql.Connection;
5	import java.sql.DriverManager;
6	import java.sql.PreparedStatement;
7	import java.sql.ResultSet;
8	import java.sql.SQLException;
9	import java.util.ArrayList;
10	import java.util.HashMap;
11	import java.util.List;
12	import java.util.Map;
13	import java.util.Properties;
14	import java.util.Queue;
15	import java.util.Set;
16	import java.util.TreeSet;
17	import java.util.concurrent.ConcurrentLinkedQueue;
18	import java.util.logging.Logger;
19
20
21
22	public class Database {
23	Logger logger = Logger.getLogger(Database.class.getName());
24
25	int batchCount = 0;
26
27	Connection conn;
28	PreparedStatement saveStmt;
29
30	List<Adresse> alleAdresser;
31	Adresse alleIkkeDaekkede[];
32
33	Map<Short,List<Adresse>> daekkedeAdresserHO = new HashMap<Short,List<Adresse>>();
34
35
36
37
38
39
40	DeduplicateHelper<String> husnrbogstavCache = new DeduplicateHelper<String>();
41	DeduplicateHelper<String> ruteCache = new DeduplicateHelper<String>();
42
43	Set<Short> postnumre = new TreeSet<Short>();
44
45	//Map<Short, List<Adresse>> ikkeDaekkedePrPost = new HashMap<Short, List<Adresse>>();
46
47
48	private HashMap<Short,BoundingBox> bbCache = new HashMap<Short,BoundingBox>();
49
50	public Database(SafeProperties conf) throws SQLException,IOException {
51	this.conn = getConnection( conf );
52
53	String sql = "INSERT INTO fulddaekning.afstand_anden_rute_ny (orgId,orgPostnr, orgAdresse,orgGadeid,orgHusnr,orgHusnrBogstav,orgRute,id,postnr,adresse,gadeid,husnr,husnrbogstav,rute,afstand,`timestamp`) "+
54	"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )";
55
56	saveStmt = conn.prepareStatement(sql);
57
58	}
59
60	public void resetResultTable() throws SQLException {
61	logger.info("Dropping old result table (if exists)");
62	String sql = "DROP TABLE IF EXISTS fulddaekning.afstand_anden_rute_ny";
63	conn.createStatement().executeUpdate(sql);
64
65	logger.info("Create new result table");
66	sql = "CREATE TABLE fulddaekning.afstand_anden_rute_ny LIKE fulddaekning.afstand_anden_rute";
67	conn.createStatement().executeUpdate(sql);
68	}
69
70	public void renameResultTables() throws SQLException {
71	Constants consts = Constants.getInstance();
72	String ext = consts.getTableExtension();
73
74	logger.info("Dropping old backup table (if exists)");
75	String sql = "DROP TABLE IF EXISTS fulddaekning.afstand_anden_rute_old" + ext;
76	conn.createStatement().executeUpdate(sql);
77
78	logger.info("Rename tables");
79	String sql2 = "RENAME TABLE fulddaekning.afstand_anden_rute" + ext + " TO fulddaekning.afstand_anden_rute_old" + ext + ", fulddaekning.afstand_anden_rute_ny TO fulddaekning.afstand_anden_rute" + ext;
80
81	logger.info("Executing: " + sql2);
82	conn.createStatement().executeUpdate(sql2);
83	}
84
85	public BoundingBox getBoundingbox(short postnr) {
86	BoundingBox bb = bbCache.get(postnr);
87	return bb.clone();//never return the original / cached object
88	}
89
90
91	public Set<Short> hentPostnumreCache() {
92	return postnumre;
93	}
94
95
96	public Queue<Adresse> hentAlleIkkedaekkedeAdresser(int minPostnr, int maxPostnr) throws SQLException {
97
98	logger.info("Henter alle IKKE-daekkede adresser");
99
100	String sql = "SELECT id,a.postnr,adresse,gadeid,husnr,husnrbogstav,latitude,longitude,rute,p.distributor as ho " +
101	"FROM fulddaekning.adressetabel a " +
102	"LEFT JOIN bogleveringer.postnummerdistributor p on (a.postnr=p.postnr) " +
103	"WHERE rute IS NULL " + //Ingen dækning
104	//"AND a.postnr BETWEEN ? AND ? " +
105	"AND a.postnr BETWEEN 8000 and 8600 " +
106	//"AND a.postnr BETWEEN 9000 AND 9300 " +
107	"AND latitude IS NOT NULL " +
108	"AND longitude IS NOT NULL " +
109	"AND gadeid IS NOT NULL " +
110	"AND (a.distributor IS NULL OR a.distributor<>'LUKKET') " +
111	"LIMIT 2500";
112	PreparedStatement stmt = conn.prepareStatement(sql);
113	//stmt.setInt(1, minPostnr);
114	//stmt.setInt(2, maxPostnr);
115
116	List<Adresse> list = hentAdresseListe( stmt );
117	alleIkkeDaekkede = list.toArray( new Adresse[ list.size() ] );
118
119	logger.info("Analyserer ikke-daekkede adresser");
120
121	for (Adresse a : alleIkkeDaekkede) {
122
123
124	BoundingBox bbox;
125
126	if (! postnumre.contains(a.postnr )) {
127	postnumre.add( a.postnr );
128
129	bbox = new BoundingBox();
130
131	bbCache.put( a.postnr, bbox);
132
133	} else {
134	bbox = bbCache.get( a.postnr);
135	}
136
137	bbox.latitudeMax = Math.max(bbox.latitudeMax, a.latitude);
138	bbox.latitudeMin = Math.min(bbox.latitudeMin, a.latitude);
139	bbox.longitudeMax = Math.max(bbox.longitudeMax, a.longitude);
140	bbox.longitudeMin = Math.min(bbox.longitudeMin, a.longitude);
141
142
143	}
144	return new ConcurrentLinkedQueue<Adresse>( list );
145	}
146
147
148
149
150	public Map<Short, List<Adresse>> getDaekkedeAdresserHO() {
151	return daekkedeAdresserHO;
152	}
153
154	public int hentAlleDaekkedeAdresser() throws SQLException {
155	if ( alleAdresser == null ) {
156	String sql = "SELECT id,a.postnr,adresse,gadeid,husnr,husnrbogstav,latitude,longitude,rute,p.distributor as ho " +
157	"FROM fulddaekning.adressetabel a " +
158	"LEFT JOIN bogleveringer.postnummerdistributor p on (a.postnr=p.postnr) " +
159	"WHERE rute IS NOT NULL " +
160	"AND latitude IS NOT NULL " +
161	"AND longitude IS NOT NULL " +
162	"AND a.distributor = ? ";
163
164	// Forward only + concur_read_only + fetchsize tvinger driver til at hente en række af gangen (bedre performance ved store result sets)
165	// Se http://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html
166	PreparedStatement stmt = conn.prepareStatement(sql, java.sql.ResultSet.TYPE_FORWARD_ONLY, java.sql.ResultSet.CONCUR_READ_ONLY);
167	stmt.setFetchSize(Integer.MIN_VALUE);
168
169	stmt.setString(1, LookupMain.distributor);
170
171	List<Adresse> list = hentAdresseListe( stmt );
172
173	alleAdresser = new ArrayList<Adresse>();
174	alleAdresser.addAll(list);
175
176	for(Adresse addr : list) {
177	short ho = addr.ho;
178	List<Adresse> hoListe = daekkedeAdresserHO.get(ho);
179	if ( hoListe == null) {
180	hoListe = new ArrayList<Adresse>();
181	daekkedeAdresserHO.put(ho, hoListe);
182	}
183
184	hoListe.add(addr);
185
186	}
187	}
188
189	return alleAdresser.size();
190	}
191
192
193
194	public synchronized void gemResultat(Adresse orgAdresse, Adresse bedsteAdresse, double bedsteAfstand) throws SQLException {
195	/*String sql = "INSERT INTO fulddaekning.afstand_anden_rute_thn (orgId,orgPostnr, orgAdresse,orgGadeid,orgHusnr,orgHusnrBogstav,orgLatitude,orgLongitude,orgRute,id,postnr,adresse,gadeid,husnr,husnrbogstav,latitude,longitude,rute,afstand,`timestamp`) "+
196	"VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )";
197
198	PreparedStatement saveStmt = conn.prepareStatement(sql);*/
199
200	saveStmt.setInt(1, orgAdresse.id);
201	saveStmt.setShort(2, orgAdresse.postnr);
202	saveStmt.setString(3, orgAdresse.adresse);
203	saveStmt.setInt(4, orgAdresse.gadeid);
204	saveStmt.setShort(5, orgAdresse.husnr);
205	saveStmt.setString(6, orgAdresse.husnrbogstav);
206	saveStmt.setString(7, orgAdresse.rute);
207
208
209	saveStmt.setInt(8, bedsteAdresse.id);
210	saveStmt.setShort(9, bedsteAdresse.postnr);
211	saveStmt.setString(10, bedsteAdresse.adresse);
212	saveStmt.setInt(11, bedsteAdresse.gadeid);
213	saveStmt.setShort(12, bedsteAdresse.husnr);
214	saveStmt.setString(13, bedsteAdresse.husnrbogstav);
215	saveStmt.setString(14, bedsteAdresse.rute);
216
217	saveStmt.setDouble(15, bedsteAfstand);
218
219	saveStmt.addBatch();
220	batchCount++;
221	if (batchCount >= 1000) {
222	saveStmt.executeBatch();
223	batchCount = 0;
224	}
225	//saveStmt.executeUpdate();
226	//saveStmt.clearParameters();
227
228	//saveStmt.close();
229	}
230
231	public synchronized void saveBatch() throws SQLException{
232	saveStmt.executeBatch();
233	batchCount = 0;
234	}
235
236
237
238	protected ArrayList<Adresse> hentAdresseListe(PreparedStatement stmt) throws SQLException{
239	ArrayList<Adresse> list = new ArrayList<Adresse>( 1000000 );
240
241	//logger.info("Starting query");
242	ResultSet res = stmt.executeQuery();
243	//logger.info("Starting exec query done");
244
245	while (res.next()) {
246
247	double latitude = res.getDouble(7);
248	double longitude = res.getDouble(8);
249
250
251	Adresse adr = new Adresse(latitude,longitude);
252
253	adr.id = res.getInt(1);
254	adr.postnr = res.getShort(2);
255	adr.adresse = res.getString(3);
256	adr.gadeid = res.getInt(4);
257	adr.husnr = res.getShort(5);
258	adr.husnrbogstav = husnrbogstavCache.getInstance( res.getString(6) );
259	//adr.latitude = res.getDouble(7);
260	//adr.longitude = res.getDouble(8);
261	adr.rute = ruteCache.getInstance( res.getString(9) );
262	adr.ho = res.getShort(10);
263
264	list.add(adr);
265
266	//logger.info( "Adress:" + adr);
267	}
268
269	res.close();
270	stmt.close();
271
272	return list;
273	}
274
275	public Connection getConnection(SafeProperties conf) throws SQLException, IOException {
276
277	String db_host = conf.getSafeProperty("DB_HOST");
278	String db_user = conf.getSafeProperty("DB_USER");
279	String db_pass = conf.getSafeProperty("DB_PASS");
280
281
282
283
284	Connection conn = null;
285	Properties connectionProps = new Properties();
286	connectionProps.put("user", db_user);
287	connectionProps.put("password", db_pass);
288
289	//For debug output, tilføj denne til JDBC url'en: &profileSQL=true
290	conn = DriverManager.getConnection(
291	"jdbc:mysql://" +
292	db_host +
293	":3306/?rewriteBatchedStatements=true",
294	connectionProps);
295	logger.info("Connected to database");
296	return conn;
297	}
298
299
300	}