package dk.daoas.adressevedligehold.fileupload; import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.TreeMap; import org.apache.commons.lang3.StringUtils; import dk.daoas.adressevedligehold.beans.Address; import dk.daoas.adressevedligehold.beans.Address.AddressState; import dk.daoas.adressevedligehold.db.DatabaseCoverageUpdate; import dk.daoas.adressevedligehold.fileupload.AddressSourceEntry.EntryType; import dk.daoas.adressevedligehold.tasks.TaskLogger; import dk.daoas.adressevedligehold.util.DeduplicateHelper; import dk.daoas.adressevedligehold.util.MiscUtils; import dk.daoas.adressevedligehold.util.TimingHelper; /* * */ public class AddressManager { private TaskLogger logger = TaskLogger.getInstance(); private int duplicateCount; List
addressList; Map> > searchStructure; ArrayList rejectedEntries = new ArrayList(); Map unknownStreets = new TreeMap(); Map dbkBaneMap = new TreeMap(); public AddressManager() throws SQLException { DatabaseCoverageUpdate db = new DatabaseCoverageUpdate(); searchStructure = new TreeMap> >(); addressList = db.getAllAdresses(); DeduplicateHelper intHelper = new DeduplicateHelper(); DeduplicateHelper shortHelper = new DeduplicateHelper(); List> arraylistCache = new ArrayList>(); TimingHelper timer = new TimingHelper(); for (Address a : addressList) { Integer gadeid = intHelper.getInstance( a.gadeid ); Short husnr = shortHelper.getInstance( a.husnr ); TreeMap> gade = searchStructure.get( gadeid ); if (a.dbkBane > 0 && a.postnr<=4999) { Short bane = dbkBaneMap.get(a.postnr); if (bane == null) { dbkBaneMap.put(a.postnr, a.dbkBane); } } if (gade == null) { gade = new TreeMap>(); searchStructure.put(gadeid, gade); } ArrayList
litraList = gade.get(husnr); if (litraList == null) { litraList = new ArrayList
(); gade.put(husnr, litraList); arraylistCache.add(litraList); } litraList.add(a); } for (ArrayList
list : arraylistCache) { list.trimToSize(); } logger.info("AddressManager ready, elapsed " + timer.getElapsed() + "ms"); } @SuppressWarnings("PMD.CollapsibleIfStatements") public void closeUnvisitedAddresses(String distributor, EntryUgedage ugedage) { for (Address addr : addressList) { if (addr.distributor == null) continue; if (addr.distributor.equals(distributor) == false)//irrelevant for denne indlæsning continue; int closedCount = 0; if (addr.visitedMan == false && ugedage.mandag) { if (addr.ruteMandag != null) { addr.ruteMandag = null; addr.korelisteMandag = null; addr.stateMan = AddressState.CLOSED; closedCount++; } } if (addr.visitedTir == false && ugedage.tirsdag) { if (addr.ruteTirsdag != null) { addr.ruteTirsdag = null; addr.korelisteTirsdag = null; addr.stateTir = AddressState.CLOSED; closedCount++; } } if (addr.visitedOns == false && ugedage.onsdag) { if (addr.ruteOnsdag != null) { addr.ruteOnsdag = null; addr.korelisteOnsdag = null; addr.stateOns = AddressState.CLOSED; closedCount++; } } if (addr.visitedTor == false && ugedage.torsdag) { if (addr.ruteTorsdag != null) { addr.ruteTorsdag = null; addr.korelisteTorsdag = null; addr.stateTor = AddressState.CLOSED; closedCount++; } } if (addr.visitedFre == false && ugedage.fredag) { if (addr.ruteFredag != null) { addr.ruteFredag = null; addr.korelisteFredag = null; addr.stateFre = AddressState.CLOSED; closedCount++; } } if (addr.visitedLor == false && ugedage.lordag) { if (addr.ruteLordag != null) { addr.ruteLordag = null; addr.korelisteLordag = null; addr.stateLor = AddressState.CLOSED; closedCount++; } } if (addr.visitedSon == false && ugedage.sondag) { if (addr.ruteSondag != null) { addr.ruteSondag = null; addr.korelisteSondag = null; addr.stateSon = AddressState.CLOSED; closedCount++; } } if (addr.state == AddressState.NOT_CHANGED && closedCount>0) { addr.state = AddressState.CLOSED; } } } public void dumpChanged() { int count = 0; for (Address addr : addressList) { if (addr.state == AddressState.NOT_CHANGED) { continue; } /*if (addr.stateMan == AddressState.NOT_CHANGED || addr.stateMan == AddressState.CREATED) { continue; }*/ if (addr.stateMan != AddressState.MODIFIED ) { continue; } logger.info(addr.state + ": " + addr.toStringExtended() ); logger.info(addr.getRuteString() ); if (count++ > 50) return; } } public void writeBackChanges() { logger.info("Writing back changes"); TimingHelper timer = new TimingHelper(); List
updates = new ArrayList
(1024*1024); List
inserts = new ArrayList
(16*1024); for (Address addr : addressList) { if (addr.state == AddressState.CREATED) { inserts.add(addr); } if (addr.state == AddressState.MODIFIED || addr.state == AddressState.OPENED || addr.state == AddressState.CLOSED) { updates.add(addr); } } DatabaseCoverageUpdate db = new DatabaseCoverageUpdate(); db.updateAddresses(updates); db.saveNewAddresses(inserts); logger.info("Writeback done: elapsed " + timer.getElapsed() + "ms."); } public void visit(AddressSourceEntry entry) { if (entry.type == EntryType.TypeSingleAddress) { visitSingle(entry); } else { visitRange(entry); } } private void visitSingle(AddressSourceEntry entry) { TreeMap> gade = searchStructure.get( entry.gadeid ); if (gade == null) { if (! unknownStreets.containsKey(entry.gadeid)) { unknownStreets.put(entry.gadeid, entry.vejnavn); } createFromEntry(entry); // if we get here there was no match - so we need to create it return; } ArrayList
litraList = gade.get(entry.husnr); if (litraList == null) { createFromEntry(entry); // if we get here there was no match - so we need to create it return; } boolean found = false; for (Address addr : litraList) { if (addr.husnrbogstav.equals(entry.litra) ) { updateAddress(addr, entry); found = true; // 1 visit should be enough but as long as there's duplicates on gadeid+husnr+litra we will visit them all //when the issue with duplicates is resolved this should be reverted to a return or break } } if (found == false) { createFromEntry(entry); // if we get here there was no match - so we need to create it } } @SuppressWarnings("PMD.CollapsibleIfStatements") private void visitRange(AddressSourceEntry entry) { TreeMap> gade = searchStructure.get( entry.gadeid ); if (gade == null) { if (! unknownStreets.containsKey(entry.gadeid)) { unknownStreets.put(entry.gadeid, entry.vejnavn); } logger.info("[Range] Ukendt gadeID " + entry); return; } for (Entry> husnrEntry: gade.entrySet()) { short husnummer = husnrEntry.getKey(); if ( (entry.husnr % 2) != (husnummer%2) ){ //lige/ulige skal passe sammen continue; } if ( husnummer < entry.husnr || husnummer > entry.tilHusnr) { continue; } ArrayList
litraList = husnrEntry.getValue(); /* a=adressetabel u=input * and (a.husnr>u.FraHusNr OR (a.husnr=u.FraHusNr AND a.HusnrBogstav >= u.FraBog)) * AND (a.husnr entry.husnr || (addr.husnr == entry.husnr && addr.husnrbogstav.compareTo(entry.litra) >= 0 )) { //Using nested IF instead of && if (addr.husnr < entry.tilHusnr || (addr.husnr == entry.tilHusnr && addr.husnrbogstav.compareTo(entry.tilLitra) <= 0 )) { updateAddress(addr, entry); } } } } } private void updateAddress(Address addr, AddressSourceEntry entry) { if ( addr.distributor != null && addr.distributor.equals(entry.distributor) == false) { rejectedEntries.add(entry); logger.info("Afviser " + entry); return; } if (entry.ugedage.mandag) { if (addr.visitedMan == false) { addr.visitedMan = true; if (addr.ruteMandag == null) { addr.stateMan = AddressState.OPENED; addr.ruteMandag = entry.rute; addr.korelisteMandag = entry.koreliste; } else { if (StringUtils.equals(addr.ruteMandag, entry.rute) == false || StringUtils.equals(addr.korelisteMandag, entry.koreliste) == false) { addr.stateMan = AddressState.MODIFIED; addr.ruteMandag = entry.rute; addr.korelisteMandag = entry.koreliste; } } } else { duplicateCount++; logger.info( "Double visit monday " + addr); return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters } } if (entry.ugedage.tirsdag) { if (addr.visitedTir == false) { addr.visitedTir = true; if (addr.ruteTirsdag == null) { addr.stateTir = AddressState.OPENED; addr.ruteTirsdag = entry.rute; addr.korelisteTirsdag = entry.koreliste; } else { if (StringUtils.equals(addr.ruteTirsdag, entry.rute) == false || StringUtils.equals(addr.korelisteTirsdag, entry.koreliste) == false) { addr.stateTir = AddressState.MODIFIED; addr.ruteTirsdag = entry.rute; addr.korelisteTirsdag = entry.koreliste; } } } else { duplicateCount++; logger.info( "Double visit tuesday " + addr); return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters } } if (entry.ugedage.onsdag) { if (addr.visitedOns == false) { addr.visitedOns = true; if (addr.ruteOnsdag == null) { addr.stateOns = AddressState.OPENED; addr.ruteOnsdag = entry.rute; addr.korelisteOnsdag = entry.koreliste; } else { if (StringUtils.equals(addr.ruteOnsdag, entry.rute) == false || StringUtils.equals(addr.korelisteOnsdag, entry.koreliste) == false) { addr.stateOns = AddressState.MODIFIED; addr.ruteOnsdag = entry.rute; addr.korelisteOnsdag = entry.koreliste; } } } else { duplicateCount++; logger.info( "Double visit wednesday " + addr); return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters } } if (entry.ugedage.torsdag) { if (addr.visitedTor == false) { addr.visitedTor = true; if (addr.ruteTorsdag == null) { addr.stateTor = AddressState.OPENED; addr.ruteTorsdag = entry.rute; addr.korelisteTorsdag = entry.koreliste; } else { if (StringUtils.equals(addr.ruteTorsdag, entry.rute) == false || StringUtils.equals(addr.korelisteTorsdag, entry.koreliste) == false) { addr.stateTor = AddressState.MODIFIED; addr.ruteTorsdag = entry.rute; addr.korelisteTorsdag = entry.koreliste; } } } else { duplicateCount++; logger.info( "Double visit thursday " + addr); return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters } } if (entry.ugedage.fredag) { if (addr.visitedFre == false) { addr.visitedFre = true; if (addr.ruteFredag == null) { addr.stateFre = AddressState.OPENED; addr.ruteFredag = entry.rute; addr.korelisteFredag = entry.koreliste; } else { if (StringUtils.equals(addr.ruteFredag, entry.rute) == false || StringUtils.equals(addr.korelisteFredag, entry.koreliste) == false) { addr.stateFre = AddressState.MODIFIED; addr.ruteFredag = entry.rute; addr.korelisteFredag = entry.koreliste; } } } else { duplicateCount++; logger.info( "Double visit friday " + addr); return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters } } if (entry.ugedage.lordag) { if (addr.visitedLor == false) { addr.visitedLor = true; if (addr.ruteLordag == null) { addr.stateLor = AddressState.OPENED; addr.ruteLordag = entry.rute; addr.korelisteLordag = entry.koreliste; } else { if (StringUtils.equals(addr.ruteLordag, entry.rute) == false || StringUtils.equals(addr.korelisteLordag, entry.koreliste) == false) { addr.stateLor = AddressState.MODIFIED; addr.ruteLordag = entry.rute; addr.korelisteLordag = entry.koreliste; } } } else { duplicateCount++; logger.info( "Double visit saturday " + addr); return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters } } if (entry.ugedage.sondag) { if (addr.visitedSon == false) { addr.visitedSon = true; if (addr.ruteSondag == null) { addr.stateSon = AddressState.OPENED; addr.ruteSondag = entry.rute; addr.korelisteSondag = entry.koreliste; } else { if (StringUtils.equals(addr.ruteSondag, entry.rute) == false || StringUtils.equals(addr.korelisteSondag, entry.koreliste) == false) { addr.stateSon = AddressState.MODIFIED; addr.ruteSondag = entry.rute; addr.korelisteSondag = entry.koreliste; } } } else { duplicateCount++; logger.info( "Double visit sunday " + addr); return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters } } //addr.visited = true; if (addr.state == AddressState.NOT_CHANGED) { if (addr.distributor == null) { addr.state = AddressState.OPENED; addr.distributor = entry.distributor; } else { if (addr.stateMan != AddressState.NOT_CHANGED || addr.stateTir != AddressState.NOT_CHANGED || addr.stateOns != AddressState.NOT_CHANGED || addr.stateTor != AddressState.NOT_CHANGED || addr.stateFre != AddressState.NOT_CHANGED || addr.stateLor != AddressState.NOT_CHANGED || addr.stateSon != AddressState.NOT_CHANGED ){ addr.state = AddressState.MODIFIED; } } } updateDbkBane(addr); } private void createFromEntry(AddressSourceEntry entry) { if (entry.husnr >= 1000) { //husnumre i DK må kun være på 3 cifre return; } if (entry.husnr == 999 || entry.husnr == 998) { //frasorter de to højest gyldige husnr - det er typisk special adresser return; } if (entry.litra.length() > 1) { //litra må kun være på 1 tegn return; } if (entry.litra.length() == 1) { char litra = entry.litra.charAt(0); if ( Character.isLetter(litra) == false ) { return; } } if (entry.kommunekode == 0 || entry.vejkode == 0) { return; } if (entry.kommunekode < 100) { return; } if (entry.vejkode > 9999) { return; } if (entry.gadeid == 0) { return; } String gadeidStr = Integer.toString(entry.gadeid); if (gadeidStr.length() != 9) { return; } if (Short.parseShort(gadeidStr.substring(0, 4)) != entry.postnr) { return; //gadeid / postnr mismatch } logger.info("Opretter adresse ud fra " + entry); Address addr = new Address(); addr.state = AddressState.CREATED; addr.distributor = entry.distributor; addr.gadeid = entry.gadeid; addr.kommunekode = entry.kommunekode; addr.vejkode = entry.vejkode; addr.vejnavn = entry.vejnavn; addr.husnr = entry.husnr; addr.husnrbogstav = entry.litra; addr.postnr = entry.postnr; updateAddress(addr, entry); // Nu er det nye adresse object oprettet - nu skal det gemmes i søge strukturen og totallisten TreeMap> gade = searchStructure.get( addr.gadeid ); if (gade == null) { gade = new TreeMap>(); searchStructure.put(addr.gadeid, gade); } ArrayList
litraList = gade.get(addr.husnr); if (litraList == null) { litraList = new ArrayList
(); gade.put(addr.husnr, litraList); } litraList.add(addr); addressList.add(addr); } /* DAO: UPDATE fulddaekning.adressetabel SET dbkbane = case when substr(korelisteMa,1,2) IN ('07','10','11','12','14','15','16','18','19','20') then 205 #DAO-BRA when substr(korelisteMa,1,2) BETWEEN 24 and 30 then 201 #DAO-Ovrige else 202 #dao syd (52-99= END WHERE distributor='DAO' AND ruteMa is not null; FD: Altid 200 NS: Altid 204 BK: 195,196,197,198 */ private void updateDbkBane(Address a) { List daoBane205 = Arrays.asList( new Short[] {7,10,11,12,12,14,15,16,18,19,20} ); Short bane = null; switch (a.distributor) { case "BK": bane = dbkBaneMap.get(a.postnr); break; case "DAO": String koreliste = MiscUtils.firstNonNull(a.korelisteMandag,a.korelisteLordag, a.korelisteSondag); //DAO har kun 3 dækningstyper short first2 = Short.parseShort( koreliste.substring(0,2) ); if ( daoBane205.contains(first2) ) { bane = 205; } else if (first2 >= 24 && first2<=30) { bane = 201; } else { bane = 202; } break; case "FD": bane = 200; break; case "NS": bane = 204; break; default: throw new RuntimeException("Ukendt distributor: " + a.distributor); //Silence findBugs } if (bane == null) { throw new RuntimeException("Ukendt bane for postnr: " + a.postnr); } if (a.dbkBane != bane) { a.dbkBane = bane; if (a.state ==AddressState.NOT_CHANGED) { a.state = AddressState.MODIFIED; } } } public List
getAddressList() { return Collections.unmodifiableList(addressList); } public Map getUnknownStreets() { return Collections.unmodifiableMap( unknownStreets ); } public int getDuplicateCount() { return duplicateCount; } public int getRejectedCount() { return rejectedEntries.size(); } }