--- dao/DaoAdresseVedligehold/src/main/java/dk/daoas/adressevedligehold/AddressManager.java 2016/01/25 10:25:22 2840 +++ dao/DaoAdresseVedligehold/src/main/java/dk/daoas/adressevedligehold/fileupload/AddressManager.java 2016/02/13 15:45:02 2947 @@ -1,37 +1,50 @@ -package dk.daoas.adressevedligehold; +package dk.daoas.adressevedligehold.fileupload; import java.sql.SQLException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.TreeMap; import org.apache.commons.lang3.StringUtils; -import dk.daoas.adressevedligehold.AddressSourceEntry.EntryType; import dk.daoas.adressevedligehold.beans.Address; import dk.daoas.adressevedligehold.beans.Address.AddressState; -import dk.daoas.adressevedligehold.db.DatabaseLayerImplementation; +import dk.daoas.adressevedligehold.db.DatabaseCoverageUpdate; +import dk.daoas.adressevedligehold.fileupload.AddressSourceEntry.EntryType; +import dk.daoas.adressevedligehold.tasks.TaskLogger; import dk.daoas.adressevedligehold.util.DeduplicateHelper; +import dk.daoas.adressevedligehold.util.MiscUtils; import dk.daoas.adressevedligehold.util.TimingHelper; /* - * TODO: implementer visitRange() * TODO: håndtering af entry dupletter ! (+ rapportering af dem) + + + */ public class AddressManager { + private TaskLogger logger = TaskLogger.getInstance(); + + private int duplicateCount; List
addressList; Map> > searchStructure; - ArrayList afvisteEntries = new ArrayList(); + ArrayList rejectedEntries = new ArrayList(); + + Map unknownStreets = new TreeMap(); + + Map dbkBaneMap = new TreeMap(); public AddressManager() throws SQLException { - DatabaseLayerImplementation db = new DatabaseLayerImplementation(); + DatabaseCoverageUpdate db = new DatabaseCoverageUpdate(); searchStructure = new TreeMap> >(); @@ -49,6 +62,13 @@ Short husnr = shortHelper.getInstance( a.husnr ); TreeMap> gade = searchStructure.get( gadeid ); + if (a.dbkBane > 0 && a.postnr<=4999) { + Short bane = dbkBaneMap.get(a.postnr); + if (bane == null) { + dbkBaneMap.put(a.postnr, a.dbkBane); + } + } + if (gade == null) { gade = new TreeMap>(); searchStructure.put(gadeid, gade); @@ -68,11 +88,12 @@ list.trimToSize(); } - System.out.println("AddressManager ready, elapsed " + timer.getElapsed() + "ms"); + logger.info("AddressManager ready, elapsed " + timer.getElapsed() + "ms"); } - + + @SuppressWarnings("PMD.CollapsibleIfStatements") public void closeUnvisitedAddresses(String distributor, EntryUgedage ugedage) { for (Address addr : addressList) { if (addr.distributor == null) @@ -157,24 +178,24 @@ if (addr.state == AddressState.NOT_CHANGED) { continue; } - if (addr.stateMan == AddressState.NOT_CHANGED || addr.stateMan == AddressState.CREATED) { + /*if (addr.stateMan == AddressState.NOT_CHANGED || addr.stateMan == AddressState.CREATED) { continue; - } + }*/ if (addr.stateMan != AddressState.MODIFIED ) { continue; } - System.out.println(addr.state + ": " + addr.toStringExtended() ); - System.out.println(addr.getRuteString() ); + logger.info(addr.state + ": " + addr.toStringExtended() ); + logger.info(addr.getRuteString() ); - if (count++ > 15) + if (count++ > 50) return; } } public void writeBackChanges() { - System.out.println("Writing back changes"); + logger.info("Writing back changes"); TimingHelper timer = new TimingHelper(); List
updates = new ArrayList
(1024*1024); @@ -189,10 +210,14 @@ } } - DatabaseLayerImplementation db = new DatabaseLayerImplementation(); + DatabaseCoverageUpdate db = new DatabaseCoverageUpdate(); + db.updateAddresses(updates); + db.saveNewAddresses(inserts); + + - System.out.println("Writeback done: elapsed " + timer.getElapsed() + "ms."); + logger.info("Writeback done: elapsed " + timer.getElapsed() + "ms."); } @@ -207,6 +232,10 @@ private void visitSingle(AddressSourceEntry entry) { TreeMap> gade = searchStructure.get( entry.gadeid ); if (gade == null) { + if (! unknownStreets.containsKey(entry.gadeid)) { + unknownStreets.put(entry.gadeid, entry.vejnavn); + } + createFromEntry(entry); // if we get here there was no match - so we need to create it return; } @@ -216,238 +245,413 @@ createFromEntry(entry); // if we get here there was no match - so we need to create it return; } - + boolean found = false; for (Address addr : litraList) { if (addr.husnrbogstav.equals(entry.litra) ) { updateAddress(addr, entry); - return; //Done for now - no need to look at the rest of the list + found = true; // 1 visit should be enough but as long as there's duplicates on gadeid+husnr+litra we will visit them all + //when the issue with duplicates is resolved this should be reverted to a return or break } } - - createFromEntry(entry); // if we get here there was no match - so we need to create it + if (found == false) { + createFromEntry(entry); // if we get here there was no match - so we need to create it + } } + @SuppressWarnings("PMD.CollapsibleIfStatements") private void visitRange(AddressSourceEntry entry) { - throw new UnsupportedOperationException("Not implemented yet"); + TreeMap> gade = searchStructure.get( entry.gadeid ); + if (gade == null) { + if (! unknownStreets.containsKey(entry.gadeid)) { + unknownStreets.put(entry.gadeid, entry.vejnavn); + } + + logger.info("[Range] Ukendt gadeID " + entry); + return; + } + + for (Entry> husnrEntry: gade.entrySet()) { + + short husnummer = husnrEntry.getKey(); + + if ( (entry.husnr % 2) != (husnummer%2) ){ //lige/ulige skal passe sammen + continue; + } + + if ( husnummer < entry.husnr || husnummer > entry.tilHusnr) { + continue; + } + + ArrayList
litraList = husnrEntry.getValue(); + + /* a=adressetabel u=input + * and (a.husnr>u.FraHusNr OR (a.husnr=u.FraHusNr AND a.HusnrBogstav >= u.FraBog)) + * AND (a.husnr entry.husnr || (addr.husnr == entry.husnr && addr.husnrbogstav.compareTo(entry.litra) >= 0 )) { + //Using nested IF instead of && + if (addr.husnr < entry.tilHusnr || (addr.husnr == entry.tilHusnr && addr.husnrbogstav.compareTo(entry.tilLitra) <= 0 )) { + updateAddress(addr, entry); + } + } + } + } } private void updateAddress(Address addr, AddressSourceEntry entry) { if ( addr.distributor != null && addr.distributor.equals(entry.distributor) == false) { - afvisteEntries.add(entry); - System.out.println("Afviser " + entry); + rejectedEntries.add(entry); + logger.info("Afviser " + entry); return; } - addr.visited = true; - if (addr.state == AddressState.NOT_CHANGED) { - if (addr.distributor == null) { - addr.state = AddressState.OPENED; - addr.distributor = entry.distributor; - } else { - addr.state = AddressState.MODIFIED; - } - } + if (entry.ugedage.mandag) { - addr.visitedMan = true; - if (addr.ruteMandag == null) { - addr.stateMan = AddressState.OPENED; - addr.ruteMandag = entry.rute; - addr.korelisteMandag = entry.koreliste; - } else { - - if (StringUtils.equals(addr.ruteMandag, entry.rute) == false || StringUtils.equals(addr.korelisteMandag, entry.koreliste) == false) { - addr.stateMan = AddressState.MODIFIED; + if (addr.visitedMan == false) { + addr.visitedMan = true; + if (addr.ruteMandag == null) { + addr.stateMan = AddressState.OPENED; addr.ruteMandag = entry.rute; - addr.korelisteMandag = entry.koreliste; - } + addr.korelisteMandag = entry.koreliste; + } else { + + if (StringUtils.equals(addr.ruteMandag, entry.rute) == false || StringUtils.equals(addr.korelisteMandag, entry.koreliste) == false) { + addr.stateMan = AddressState.MODIFIED; + addr.ruteMandag = entry.rute; + addr.korelisteMandag = entry.koreliste; + } + } + } else { + duplicateCount++; + logger.info( "Double visit monday " + addr); + return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters } + } if (entry.ugedage.tirsdag) { - addr.visitedTir = true; - if (addr.ruteTirsdag == null) { - addr.stateTir = AddressState.OPENED; - addr.ruteTirsdag = entry.rute; - addr.korelisteTirsdag = entry.koreliste; - } else { - - if (StringUtils.equals(addr.ruteTirsdag, entry.rute) == false || StringUtils.equals(addr.korelisteTirsdag, entry.koreliste) == false) { - addr.stateTir = AddressState.MODIFIED; + if (addr.visitedTir == false) { + addr.visitedTir = true; + if (addr.ruteTirsdag == null) { + addr.stateTir = AddressState.OPENED; addr.ruteTirsdag = entry.rute; - addr.korelisteTirsdag = entry.koreliste; + addr.korelisteTirsdag = entry.koreliste; + } else { + + if (StringUtils.equals(addr.ruteTirsdag, entry.rute) == false || StringUtils.equals(addr.korelisteTirsdag, entry.koreliste) == false) { + addr.stateTir = AddressState.MODIFIED; + addr.ruteTirsdag = entry.rute; + addr.korelisteTirsdag = entry.koreliste; + } } - } + } else { + duplicateCount++; + logger.info( "Double visit tuesday " + addr); + return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters + } } if (entry.ugedage.onsdag) { - addr.visitedOns = true; - if (addr.ruteOnsdag == null) { - addr.stateOns = AddressState.OPENED; - addr.ruteOnsdag = entry.rute; - addr.korelisteOnsdag = entry.koreliste; - } else { - - if (StringUtils.equals(addr.ruteOnsdag, entry.rute) == false || StringUtils.equals(addr.korelisteOnsdag, entry.koreliste) == false) { - addr.stateOns = AddressState.MODIFIED; + if (addr.visitedOns == false) { + addr.visitedOns = true; + if (addr.ruteOnsdag == null) { + addr.stateOns = AddressState.OPENED; addr.ruteOnsdag = entry.rute; - addr.korelisteOnsdag = entry.koreliste; + addr.korelisteOnsdag = entry.koreliste; + } else { + + if (StringUtils.equals(addr.ruteOnsdag, entry.rute) == false || StringUtils.equals(addr.korelisteOnsdag, entry.koreliste) == false) { + addr.stateOns = AddressState.MODIFIED; + addr.ruteOnsdag = entry.rute; + addr.korelisteOnsdag = entry.koreliste; + } } - } + } else { + duplicateCount++; + logger.info( "Double visit wednesday " + addr); + return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters + } } if (entry.ugedage.torsdag) { - addr.visitedTor = true; - if (addr.ruteTorsdag == null) { - addr.stateTor = AddressState.OPENED; - addr.ruteTorsdag = entry.rute; - addr.korelisteTorsdag = entry.koreliste; - } else { - - - if (StringUtils.equals(addr.ruteTorsdag, entry.rute) == false || StringUtils.equals(addr.korelisteTorsdag, entry.koreliste) == false) { - addr.stateTor = AddressState.MODIFIED; + if (addr.visitedTor == false) { + addr.visitedTor = true; + if (addr.ruteTorsdag == null) { + addr.stateTor = AddressState.OPENED; addr.ruteTorsdag = entry.rute; - addr.korelisteTorsdag = entry.koreliste; + addr.korelisteTorsdag = entry.koreliste; + } else { + + + if (StringUtils.equals(addr.ruteTorsdag, entry.rute) == false || StringUtils.equals(addr.korelisteTorsdag, entry.koreliste) == false) { + addr.stateTor = AddressState.MODIFIED; + addr.ruteTorsdag = entry.rute; + addr.korelisteTorsdag = entry.koreliste; + } } - } + } else { + duplicateCount++; + logger.info( "Double visit thursday " + addr); + return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters + } } if (entry.ugedage.fredag) { - addr.visitedFre = true; - if (addr.ruteFredag == null) { - addr.stateFre = AddressState.OPENED; - addr.ruteFredag = entry.rute; - addr.korelisteFredag = entry.koreliste; - } else { - - if (StringUtils.equals(addr.ruteFredag, entry.rute) == false || StringUtils.equals(addr.korelisteFredag, entry.koreliste) == false) { - addr.stateFre = AddressState.MODIFIED; + if (addr.visitedFre == false) { + addr.visitedFre = true; + if (addr.ruteFredag == null) { + addr.stateFre = AddressState.OPENED; addr.ruteFredag = entry.rute; - addr.korelisteFredag = entry.koreliste; + addr.korelisteFredag = entry.koreliste; + } else { + + if (StringUtils.equals(addr.ruteFredag, entry.rute) == false || StringUtils.equals(addr.korelisteFredag, entry.koreliste) == false) { + addr.stateFre = AddressState.MODIFIED; + addr.ruteFredag = entry.rute; + addr.korelisteFredag = entry.koreliste; + } } - } + } else { + duplicateCount++; + logger.info( "Double visit friday " + addr); + return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters + } } if (entry.ugedage.lordag) { - addr.visitedLor = true; - if (addr.ruteLordag == null) { - addr.stateLor = AddressState.OPENED; - addr.ruteLordag = entry.rute; - addr.korelisteLordag = entry.koreliste; - } else { - - - if (StringUtils.equals(addr.ruteLordag, entry.rute) == false || StringUtils.equals(addr.korelisteLordag, entry.koreliste) == false) { - addr.stateLor = AddressState.MODIFIED; + if (addr.visitedLor == false) { + addr.visitedLor = true; + if (addr.ruteLordag == null) { + addr.stateLor = AddressState.OPENED; addr.ruteLordag = entry.rute; - addr.korelisteLordag = entry.koreliste; + addr.korelisteLordag = entry.koreliste; + } else { + + + if (StringUtils.equals(addr.ruteLordag, entry.rute) == false || StringUtils.equals(addr.korelisteLordag, entry.koreliste) == false) { + addr.stateLor = AddressState.MODIFIED; + addr.ruteLordag = entry.rute; + addr.korelisteLordag = entry.koreliste; + } } - } + } else { + duplicateCount++; + logger.info( "Double visit saturday " + addr); + return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters + } } if (entry.ugedage.sondag) { - addr.visitedSon = true; - if (addr.ruteSondag == null) { - addr.stateSon = AddressState.OPENED; - addr.ruteSondag = entry.rute; - addr.korelisteSondag = entry.koreliste; - } else { - - if (StringUtils.equals(addr.ruteSondag, entry.rute) == false || StringUtils.equals(addr.korelisteSondag, entry.koreliste) == false) { - addr.stateSon = AddressState.MODIFIED; + if (addr.visitedSon == false) { + addr.visitedSon = true; + if (addr.ruteSondag == null) { + addr.stateSon = AddressState.OPENED; addr.ruteSondag = entry.rute; - addr.korelisteSondag = entry.koreliste; + addr.korelisteSondag = entry.koreliste; + } else { + + if (StringUtils.equals(addr.ruteSondag, entry.rute) == false || StringUtils.equals(addr.korelisteSondag, entry.koreliste) == false) { + addr.stateSon = AddressState.MODIFIED; + addr.ruteSondag = entry.rute; + addr.korelisteSondag = entry.koreliste; + } } - } + } else { + duplicateCount++; + logger.info( "Double visit sunday " + addr); + return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters + } } - + + //addr.visited = true; + if (addr.state == AddressState.NOT_CHANGED) { + if (addr.distributor == null) { + addr.state = AddressState.OPENED; + addr.distributor = entry.distributor; + } else { + if (addr.stateMan != AddressState.NOT_CHANGED || addr.stateTir != AddressState.NOT_CHANGED + || addr.stateOns != AddressState.NOT_CHANGED || addr.stateTor != AddressState.NOT_CHANGED + || addr.stateFre != AddressState.NOT_CHANGED || addr.stateLor != AddressState.NOT_CHANGED || addr.stateSon != AddressState.NOT_CHANGED ){ + + addr.state = AddressState.MODIFIED; + } + } + } + + updateDbkBane(addr); + } private void createFromEntry(AddressSourceEntry entry) { - System.out.println("Opretter adresse ud fra " + entry); + if (entry.husnr >= 1000) { //husnumre i DK må kun være på 3 cifre + return; + } - Address a = new Address(); - a.state = AddressState.CREATED; + if (entry.husnr == 999) { + return; + } - a.distributor = entry.distributor; - a.gadeid = entry.gadeid; - a.kommunekode = entry.kommunekode; - a.vejkode = entry.vejkode; - a.vejnavn = entry.vejnavn; - a.husnr = entry.husnr; - a.husnrbogstav = entry.litra; - a.postnr = entry.postnr; - a.visited = true; + if (entry.litra.length() > 1) { //litra må kun være på 1 tegn + return; + } + if (entry.litra.length() == 1) { + char litra = entry.litra.charAt(0); + if ( Character.isLetter(litra) == false ) { + return; + } + + } - if (entry.ugedage.mandag) { - a.ruteMandag = entry.rute; - a.korelisteMandag = entry.koreliste; + if (entry.kommunekode == 0 || entry.vejkode == 0) { + return; } - if (entry.ugedage.tirsdag) { - a.ruteTirsdag = entry.rute; - a.korelisteTirsdag = entry.koreliste; + if (entry.kommunekode < 100) { + return; } - if (entry.ugedage.onsdag) { - a.ruteOnsdag = entry.rute; - a.korelisteOnsdag = entry.koreliste; - } - if (entry.ugedage.torsdag) { - a.ruteTorsdag = entry.rute; - a.korelisteTorsdag = entry.koreliste; - } - if (entry.ugedage.fredag) { - a.ruteFredag = entry.rute; - a.korelisteFredag = entry.koreliste; - } - if (entry.ugedage.lordag) { - a.ruteLordag = entry.rute; - a.korelisteLordag = entry.koreliste; - } - if (entry.ugedage.sondag) { - a.ruteSondag = entry.rute; - a.korelisteLordag = entry.koreliste; - } - // Nu er det nye adresse object oprettet - nu skal det gemmes i søge strukturen og totallisten + if (entry.gadeid == 0) { + return; + } + + String gadeidStr = Integer.toString(entry.gadeid); + if (gadeidStr.length() != 9) { + return; + } + if (Short.parseShort(gadeidStr.substring(0, 4)) != entry.postnr) { + return; //gadeid / postnr mismatch + } + + + + logger.info("Opretter adresse ud fra " + entry); + Address addr = new Address(); + addr.state = AddressState.CREATED; + + addr.distributor = entry.distributor; + addr.gadeid = entry.gadeid; + addr.kommunekode = entry.kommunekode; + addr.vejkode = entry.vejkode; + addr.vejnavn = entry.vejnavn; + addr.husnr = entry.husnr; + addr.husnrbogstav = entry.litra; + addr.postnr = entry.postnr; + + + updateAddress(addr, entry); - TreeMap> gade = searchStructure.get( a.gadeid ); + + // Nu er det nye adresse object oprettet - nu skal det gemmes i søge strukturen og totallisten + TreeMap> gade = searchStructure.get( addr.gadeid ); if (gade == null) { gade = new TreeMap>(); - searchStructure.put(a.gadeid, gade); + searchStructure.put(addr.gadeid, gade); } - ArrayList
litraList = gade.get(a.husnr); + ArrayList
litraList = gade.get(addr.husnr); if (litraList == null) { litraList = new ArrayList
(); - gade.put(a.husnr, litraList); + gade.put(addr.husnr, litraList); } - litraList.add(a); - addressList.add(a); + litraList.add(addr); + addressList.add(addr); } + + + /* + DAO: + UPDATE fulddaekning.adressetabel + SET dbkbane = case + when substr(korelisteMa,1,2) IN ('07','10','11','12','14','15','16','18','19','20') then 205 #DAO-BRA + when substr(korelisteMa,1,2) BETWEEN 24 and 30 then 201 #DAO-Ovrige + else 202 #dao syd (52-99= + END + WHERE distributor='DAO' + AND ruteMa is not null; + + FD: Altid 200 + NS: Altid 204 + BK: 195,196,197,198 + */ + + private void updateDbkBane(Address a) { + List daoBane205 = Arrays.asList( new Short[] {7,10,11,12,12,14,15,16,18,19,20} ); + + + Short bane = null; + + switch (a.distributor) { + case "BK": + bane = dbkBaneMap.get(a.postnr); + break; + case "DAO": + String koreliste = MiscUtils.firstNonNull(a.korelisteMandag,a.korelisteLordag, a.korelisteSondag); //DAO har kun 3 dækningstyper + short first2 = Short.parseShort( koreliste.substring(0,2) ); + if ( daoBane205.contains(first2) ) { + bane = 205; + } else if (first2 >= 24 && first2<=30) { + bane = 201; + } else { + bane = 202; + } + + break; + case "FD": + bane = 200; + break; + case "NS": + bane = 204; + break; + default: + throw new RuntimeException("Ukendt distributor: " + a.distributor); //Silence findBugs + } + + if (bane == null) { + throw new RuntimeException("Ukendt bane for postnr: " + a.postnr); + } + + if (a.dbkBane != bane) { + a.dbkBane = bane; + if (a.state ==AddressState.NOT_CHANGED) { + a.state = AddressState.MODIFIED; + } + } + } + public List
getAddressList() { return Collections.unmodifiableList(addressList); } - public int getAfvistAntal() { - return afvisteEntries.size(); + public Map getUnknownStreets() { + return Collections.unmodifiableMap( unknownStreets ); + } + + public int getDuplicateCount() { + return duplicateCount; + } + + public int getRejectedCount() { + return rejectedEntries.size(); } }