--- dao/DaoAdresseVedligehold/src/main/java/dk/daoas/adressevedligehold/dawa/DawaTask.java 2016/04/18 18:56:35 3002 +++ dao/DaoAdresseVedligehold/src/main/java/dk/daoas/adressevedligehold/dawa/DawaTask.java 2016/07/22 09:48:05 3069 @@ -1,8 +1,23 @@ package dk.daoas.adressevedligehold.dawa; -import com.google.gson.Gson; - +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.TreeMap; + +import com.google.common.base.Splitter; + +import dk.daoas.adressevedligehold.MailSender; +import dk.daoas.adressevedligehold.beans.Address; +import dk.daoas.adressevedligehold.beans.Address.AddressState; +import dk.daoas.adressevedligehold.db.DatabaseCoverageUpdate; import dk.daoas.adressevedligehold.tasks.Task; +import dk.daoas.adressevedligehold.tasks.TaskLogger; +import dk.daoas.adressevedligehold.util.CountMap; +import dk.daoas.adressevedligehold.util.DeduplicateHelper; +import dk.daoas.adressevedligehold.util.TimingHelper; /* * Alle regioner: http://dawa.aws.dk/regioner/ @@ -12,10 +27,21 @@ public class DawaTask extends Task { + final static String KOMMENTAR_TEMPLATE = "Oprettet via DAWA"; + + private TaskLogger logger = TaskLogger.getInstance(); + private short regionCode; private String regionName; - private Gson gson = new Gson(); + int antalBehandlede = 0; + int antalKommuner = 0; + + + DeduplicateHelper dedup = new DeduplicateHelper<>(); + + + public DawaTask(short regionCode, String regionName){ this.regionCode = regionCode; @@ -25,8 +51,195 @@ @Override protected void taskRun() throws Exception { - // TODO Auto-generated method stub + TimingHelper timer = new TimingHelper(); + + DatabaseCoverageUpdate db = new DatabaseCoverageUpdate(); + + DawaWebservice dawa = new DawaWebservice(); + TreeMap search = new TreeMap<>(); + + // stage 1 - hent alle data + List
adresser = db.getAllAdresses(); + + + // stage 2 opbyg søge struktur + logger.info("Building search structure"); + + + for (Address addr : adresser) { + String dav = getDAVid(addr); + DawaGade gade = search.get(dav); + if (gade == null) { + gade = new DawaGade(); + search.put(dav, gade); + } + gade.gadeids.add( addr.gadeid ); + gade.adresser.add(addr); + } + logger.info("Build search structure done"); + + Map kommuner = dawa.kommunerIRegion(regionCode); + + String subject = "DAWA indlæsning - " + regionName; + + StringBuilder headerSB = new StringBuilder(); + headerSB.append("

Indlæsnings rapport for " + regionName + "


"); + + List
totalNye = new ArrayList<>(); + List
totalFejl = new ArrayList<>(); + CountMap totalUkendtDAV = new CountMap(); + CountMap totalDobbeltDav = new CountMap(); + + + + antalKommuner = kommuner.size(); + + + StringBuilder sb = new StringBuilder(); + + for (Map.Entry entry : kommuner.entrySet()) { + int kommunekode = entry.getValue(); + String kommunenavn = entry.getKey(); + + logger.info("Behandler: " + kommunekode + " " + kommunenavn); + + + List
dawaListe = dawa.adresserIKommune(kommunekode); + // stage 3 søg + CountMap ukendteDAV = new CountMap(); + CountMap dobbeltGadeidDAV = new CountMap(); + List
nyeAdresser = new ArrayList<>(); + + for (Address dawaAddr : dawaListe) { + String dav = getDAVid(dawaAddr); + DawaGade gade = search.get(dav); + if (gade == null) { + String key = dav + " " + dawaAddr.vejnavn; + ukendteDAV.increment(key); + continue; + } + + if (gade.gadeids.size() != 1) { + String key = dav + " " + dawaAddr.vejnavn; + dobbeltGadeidDAV.increment(key); + continue; + } + + boolean fundet = false; + for (Address target : gade.adresser) { + if ( target.husnr == dawaAddr.husnr && target.husnrbogstav.equals(dawaAddr.husnrbogstav) ) { + fundet = true; + break; + } + } + + if (fundet == false) { + dawaAddr.gadeid = gade.gadeids.iterator().next(); + dawaAddr.state = AddressState.CREATED; + dawaAddr.kommentar = KOMMENTAR_TEMPLATE; + nyeAdresser.add(dawaAddr); + } + } + logger.info("Address Search done - starting save to DB"); + + db.saveNewAddresses(nyeAdresser); + + logger.info("Save to db done - adding to report mail"); + + List
saveOK = new ArrayList<>(); + List
saveFail = new ArrayList<>(); + for (Address a : nyeAdresser) { + if (a.state == AddressState.CREATE_FAIL) { + saveFail.add(a); + } else { + saveOK.add(a); + } + } + + totalNye.addAll(saveOK); + totalFejl.addAll(saveFail); + + totalUkendtDAV.putAll(ukendteDAV); + totalDobbeltDav.putAll(dobbeltGadeidDAV); + + + antalBehandlede++; + + + sb.append("

Kommune: " + kommunenavn + "

\n"); + + sb.append("Kommunekode: " + kommunekode + "
\n"); + sb.append("Antal Inserts: " + saveOK.size() + "
\n"); + sb.append("Antal insert fails: " + saveFail.size() + "
\n"); + sb.append("Antal ukendte dav: " + ukendteDAV.size() + "
\n"); + sb.append("Antal dobbelt gadeid dav: " + dobbeltGadeidDAV.size() + "
\n"); + } + + logger.info("Sendingreport mail"); + + headerSB.append("Antal Inserts: " + totalNye.size() + "
\n"); + headerSB.append("Antal insert fails: " + totalFejl.size() + "
\n"); + headerSB.append("Antal ukendte dav: " + totalUkendtDAV.size() + "
\n"); + headerSB.append("Antal dobbelt gadeid dav: " + totalDobbeltDav.size() + "
\n"); + headerSB.append("Elapsed: " + timer.getElapsed() + " ms.
\n"); + + Map attachments = new TreeMap<>(); + attachments.put("saveok.csv", buildAddressFile(totalNye) ); + attachments.put("savefailed.csv", buildAddressFile(totalFejl) ); + + attachments.put("ukendte_dav.csv", buildDavFile(totalUkendtDAV) ); + attachments.put("dobbelt_dav_gadeid.csv", buildDavFile(totalDobbeltDav) ); + + + MailSender.sendMailWithAttachments(subject, headerSB.toString() + sb.toString(), attachments); + + //Cleaning + dedup.clear(); + } + + private String buildDavFile(CountMap dav) { + StringBuilder sb = new StringBuilder(); + + sb.append("Gadeid;Vejnavn;Antal\r\n"); + for ( Entry entry : dav.entrySet()) { + + List split = Splitter.on(' ').limit(2).splitToList(entry.getKey()); + sb.append( split.get(0) ).append(";");//gadeid + sb.append( split.get(1) ).append(";");//vejnavn + + sb.append( entry.getValue() ).append("\r\n");//vejnavn + + } + + return sb.toString(); + } + + private String buildAddressFile(List
addr) { + + Collections.sort(addr); //brug dens egen compareTo funktion + + StringBuilder sb = new StringBuilder(); + + sb.append("GadeID;Postnr;Kom.kode;Vejkode;Vejnavn;Husnr;Litra\r\n"); + for (Address a: addr) { + sb.append(a.gadeid).append(";"); + sb.append(a.postnr).append(";"); + sb.append(a.kommunekode).append(";"); + sb.append(a.vejkode).append(";"); + sb.append(a.vejnavn).append(";"); + sb.append(a.husnr).append(";"); + sb.append(a.husnrbogstav).append("\r\n"); + } + return sb.toString(); + + } + + + String getDAVid(Address addr) { + String dav = String.format("%04d%03d%04d", addr.postnr, addr.kommunekode, addr.vejkode); + + return dedup.getInstance(dav); } @Override @@ -41,7 +254,12 @@ @Override public double getPercentCompleted() { - return 0; + if (antalKommuner == 0) { + return 0; + } + + return (antalBehandlede / ((double)antalKommuner) ) * 100.0; + }