package dk.daoas.fulddaekning; import java.io.IOException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; import java.util.Queue; import java.util.Set; import java.util.TreeSet; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.logging.Logger; public class Database { Logger logger = Logger.getLogger(Database.class.getName()); int batchCount = 0; Connection conn; PreparedStatement saveStmt; List alleAdresser; Adresse alleIkkeDaekkede[]; Map> daekkedeAdresserHO = new HashMap>(); DeduplicateHelper vejnavnCache = new DeduplicateHelper(); DeduplicateHelper husnrbogstavCache = new DeduplicateHelper(); DeduplicateHelper ruteCache = new DeduplicateHelper(); Set postnumre = new TreeSet(); //Map> ikkeDaekkedePrPost = new HashMap>(); private HashMap bbCache = new HashMap(); public Database(SafeProperties conf) throws SQLException,IOException { this.conn = getConnection( conf ); String sql = "INSERT INTO fulddaekning.afstand_anden_rute_ny (orgId,orgPostnr, orgAdresse,orgGadeid,orgHusnr,orgHusnrBogstav,orgRute,id,postnr,adresse,gadeid,husnr,husnrbogstav,rute,afstand,`timestamp`) "+ "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )"; saveStmt = conn.prepareStatement(sql); } public void resetResultTable() throws SQLException { logger.info("Dropping old result table (if exists)"); String sql = "DROP TABLE IF EXISTS fulddaekning.afstand_anden_rute_ny"; conn.createStatement().executeUpdate(sql); logger.info("Create new result table"); sql = "CREATE TABLE fulddaekning.afstand_anden_rute_ny LIKE fulddaekning.afstand_anden_rute"; conn.createStatement().executeUpdate(sql); } public void renameResultTables() throws SQLException { if (LookupMain.test_mode == true) { throw new RuntimeException("Can not rename tables in test mode"); } Constants consts = Constants.getInstance(); String ext = consts.getTableExtension(); logger.info("Dropping old backup table (if exists)"); String sql = "DROP TABLE IF EXISTS fulddaekning.afstand_anden_rute_old" + ext; conn.createStatement().executeUpdate(sql); logger.info("Rename tables"); String sql2 = "RENAME TABLE fulddaekning.afstand_anden_rute" + ext + " TO fulddaekning.afstand_anden_rute_old" + ext + ", fulddaekning.afstand_anden_rute_ny TO fulddaekning.afstand_anden_rute" + ext; logger.info("Executing: " + sql2); conn.createStatement().executeUpdate(sql2); } public BoundingBox getBoundingbox(short postnr) { BoundingBox bb = bbCache.get(postnr); return bb.clone();//never return the original / cached object } public Set hentPostnumreCache() { return postnumre; } public Queue hentAlleIkkedaekkedeAdresser(int minPostnr, int maxPostnr) throws SQLException { logger.info("Henter alle IKKE-daekkede adresser"); String sql = "SELECT id,a.postnr,vejnavn,gadeid,husnr,husnrbogstav,latitude,longitude,rute,p.distributor as ho " + "FROM fulddaekning.adressetabel a " + "LEFT JOIN bogleveringer.postnummerdistributor p on (a.postnr=p.postnr) " + "WHERE ruteMa IS NULL " + //Ingen dækning "AND a.postnr BETWEEN ? AND ? " + "AND latitude IS NOT NULL " + "AND longitude IS NOT NULL " + "AND gadeid IS NOT NULL " + "AND (a.distributor IS NULL OR a.distributor<>'LUKKET') " ; if (LookupMain.test_mode == true) { sql = sql + " LIMIT 100 "; } PreparedStatement stmt = conn.prepareStatement(sql); stmt.setInt(1, minPostnr); stmt.setInt(2, maxPostnr); List list = hentAdresseListe( stmt ); alleIkkeDaekkede = list.toArray( new Adresse[ list.size() ] ); logger.info("Analyserer ikke-daekkede adresser"); for (Adresse a : alleIkkeDaekkede) { BoundingBox bbox; if (! postnumre.contains(a.postnr )) { postnumre.add( a.postnr ); bbox = new BoundingBox(); bbCache.put( a.postnr, bbox); } else { bbox = bbCache.get( a.postnr); } bbox.latitudeMax = Math.max(bbox.latitudeMax, a.latitude); bbox.latitudeMin = Math.min(bbox.latitudeMin, a.latitude); bbox.longitudeMax = Math.max(bbox.longitudeMax, a.longitude); bbox.longitudeMin = Math.min(bbox.longitudeMin, a.longitude); } return new ConcurrentLinkedQueue( list ); } public Map> getDaekkedeAdresserHO() { return daekkedeAdresserHO; } public int hentAlleDaekkedeAdresser() throws SQLException { if ( alleAdresser == null ) { String sql = "SELECT id,a.postnr,vejnavn,gadeid,husnr,husnrbogstav,latitude,longitude,rute,p.distributor as ho " + "FROM fulddaekning.adressetabel a " + "LEFT JOIN bogleveringer.postnummerdistributor p on (a.postnr=p.postnr) " + "WHERE ruteMa IS NOT NULL " + "AND latitude IS NOT NULL " + "AND longitude IS NOT NULL " + "AND a.distributor = ? "; if (LookupMain.test_mode == true) { sql = sql + " AND a.postnr BETWEEN 6000 and 7200 "; } // Forward only + concur_read_only + fetchsize tvinger driver til at hente en række af gangen (bedre performance ved store result sets) // Se http://dev.mysql.com/doc/connector-j/en/connector-j-reference-implementation-notes.html PreparedStatement stmt = conn.prepareStatement(sql, java.sql.ResultSet.TYPE_FORWARD_ONLY, java.sql.ResultSet.CONCUR_READ_ONLY); stmt.setFetchSize(Integer.MIN_VALUE); stmt.setString(1, LookupMain.distributor); List list = hentAdresseListe( stmt ); alleAdresser = new ArrayList(); alleAdresser.addAll(list); for(Adresse addr : list) { short ho = addr.ho; List hoListe = daekkedeAdresserHO.get(ho); if ( hoListe == null) { hoListe = new ArrayList(); daekkedeAdresserHO.put(ho, hoListe); } hoListe.add(addr); } } return alleAdresser.size(); } public synchronized void gemResultat(Adresse orgAdresse, Adresse bedsteAdresse, double bedsteAfstand) throws SQLException { /*String sql = "INSERT INTO fulddaekning.afstand_anden_rute_thn (orgId,orgPostnr, orgVejnavn,orgGadeid,orgHusnr,orgHusnrBogstav,orgLatitude,orgLongitude,orgRute,id,postnr,vejnavn,gadeid,husnr,husnrbogstav,latitude,longitude,rute,afstand,`timestamp`) "+ "VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?, now() )"; PreparedStatement saveStmt = conn.prepareStatement(sql);*/ saveStmt.setInt(1, orgAdresse.id); saveStmt.setShort(2, orgAdresse.postnr); saveStmt.setString(3, orgAdresse.vejnavn); saveStmt.setInt(4, orgAdresse.gadeid); saveStmt.setShort(5, orgAdresse.husnr); saveStmt.setString(6, orgAdresse.husnrbogstav); saveStmt.setString(7, orgAdresse.rute); saveStmt.setInt(8, bedsteAdresse.id); saveStmt.setShort(9, bedsteAdresse.postnr); saveStmt.setString(10, bedsteAdresse.vejnavn); saveStmt.setInt(11, bedsteAdresse.gadeid); saveStmt.setShort(12, bedsteAdresse.husnr); saveStmt.setString(13, bedsteAdresse.husnrbogstav); saveStmt.setString(14, bedsteAdresse.rute); saveStmt.setDouble(15, bedsteAfstand); saveStmt.addBatch(); batchCount++; if (batchCount >= 1000) { saveStmt.executeBatch(); batchCount = 0; } //saveStmt.executeUpdate(); //saveStmt.clearParameters(); //saveStmt.close(); } public synchronized void saveBatch() throws SQLException{ saveStmt.executeBatch(); batchCount = 0; } protected ArrayList hentAdresseListe(PreparedStatement stmt) throws SQLException{ ArrayList list = new ArrayList( 1000000 ); Constants consts = Constants.getInstance(); //logger.info("Starting query"); ResultSet res = stmt.executeQuery(); //logger.info("Starting exec query done"); while (res.next()) { double latitude = res.getDouble(7); double longitude = res.getDouble(8); Adresse adr = new Adresse(latitude,longitude); adr.id = res.getInt(1); adr.postnr = res.getShort(2); adr.vejnavn = vejnavnCache.getInstance( res.getString(3) ); adr.gadeid = res.getInt(4); adr.husnr = res.getShort(5); adr.husnrbogstav = husnrbogstavCache.getInstance( res.getString(6) ); //adr.latitude = res.getDouble(7); //adr.longitude = res.getDouble(8); adr.rute = ruteCache.getInstance( res.getString(9) ); adr.ho = res.getShort(10); list.add(adr); if (consts.doCheckHO() == true && adr.ho == 0) { System.out.println( "Mangler HO" ); System.out.println( adr ); System.exit(0); } //logger.info( "Adress:" + adr); } res.close(); stmt.close(); return list; } public Connection getConnection(SafeProperties conf) throws SQLException, IOException { String db_host = conf.getSafeProperty("DB_HOST"); String db_user = conf.getSafeProperty("DB_USER"); String db_pass = conf.getSafeProperty("DB_PASS"); Connection conn = null; Properties connectionProps = new Properties(); connectionProps.put("user", db_user); connectionProps.put("password", db_pass); //For debug output, tilføj denne til JDBC url'en: &profileSQL=true conn = DriverManager.getConnection( "jdbc:mysql://" + db_host + ":3306/?rewriteBatchedStatements=true", connectionProps); logger.info("Connected to database"); return conn; } }