package dk.daoas.adressevedligehold.dawa; import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.TreeMap; import com.google.common.base.Splitter; import dk.daoas.adressevedligehold.MailSender; import dk.daoas.adressevedligehold.beans.Address; import dk.daoas.adressevedligehold.beans.Address.AddressState; import dk.daoas.adressevedligehold.db.DatabaseCoverageUpdate; import dk.daoas.adressevedligehold.tasks.Task; import dk.daoas.adressevedligehold.tasks.TaskLogger; import dk.daoas.adressevedligehold.util.CountMap; import dk.daoas.adressevedligehold.util.DeduplicateHelper; import dk.daoas.adressevedligehold.util.TimingHelper; /* * Alle regioner: http://dawa.aws.dk/regioner/ * Enkelt region: http://dawa.aws.dk/regioner/{region} */ public class DawaTask extends Task { final static String KOMMENTAR_TEMPLATE = "Oprettet via DAWA"; private TaskLogger logger = TaskLogger.getInstance(); private short regionCode; private String regionName; int antalBehandlede = 0; int antalKommuner = 0; DeduplicateHelper dedup = new DeduplicateHelper<>(); public DawaTask(short regionCode, String regionName){ this.regionCode = regionCode; this.regionName = regionName; } @Override protected void taskRun() throws Exception { TimingHelper timer = new TimingHelper(); DatabaseCoverageUpdate db = new DatabaseCoverageUpdate(); DawaWebservice dawa = new DawaWebservice(); TreeMap search = new TreeMap<>(); // stage 1 - hent alle data List
adresser = db.getAllAdresses(); // stage 2 opbyg søge struktur logger.info("Building search structure"); for (Address addr : adresser) { String dav = getDAVid(addr); DawaGade gade = search.get(dav); if (gade == null) { gade = new DawaGade(); search.put(dav, gade); } gade.gadeids.add( addr.gadeid ); gade.adresser.add(addr); } logger.info("Build search structure done"); Map kommuner = dawa.kommunerIRegion(regionCode); String subject = "DAWA indlæsning - " + regionName; StringBuilder headerSB = new StringBuilder(); headerSB.append("

Indlæsnings rapport for " + regionName + "


"); List
totalNye = new ArrayList<>(); List
totalFejl = new ArrayList<>(); CountMap totalUkendtDAV = new CountMap(); CountMap totalDobbeltDav = new CountMap(); antalKommuner = kommuner.size(); StringBuilder sb = new StringBuilder(); for (Map.Entry entry : kommuner.entrySet()) { int kommunekode = entry.getValue(); String kommunenavn = entry.getKey(); logger.info("Behandler: " + kommunekode + " " + kommunenavn); List
dawaListe = dawa.adresserIKommune(kommunekode); // stage 3 søg CountMap ukendteDAV = new CountMap(); CountMap dobbeltGadeidDAV = new CountMap(); List
nyeAdresser = new ArrayList<>(); for (Address dawaAddr : dawaListe) { String dav = getDAVid(dawaAddr); DawaGade gade = search.get(dav); if (gade == null) { String key = dav + " " + dawaAddr.vejnavn; ukendteDAV.increment(key); continue; } if (gade.gadeids.size() != 1) { String key = dav + " " + dawaAddr.vejnavn; dobbeltGadeidDAV.increment(key); continue; } boolean fundet = false; for (Address target : gade.adresser) { if ( target.husnr == dawaAddr.husnr && target.husnrbogstav.equals(dawaAddr.husnrbogstav) ) { fundet = true; break; } } if (fundet == false) { dawaAddr.gadeid = gade.gadeids.iterator().next(); dawaAddr.state = AddressState.CREATED; dawaAddr.kommentar = KOMMENTAR_TEMPLATE; nyeAdresser.add(dawaAddr); } } logger.info("Address Search done - starting save to DB"); db.saveNewAddresses(nyeAdresser); logger.info("Save to db done - adding to report mail"); List
saveOK = new ArrayList<>(); List
saveFail = new ArrayList<>(); for (Address a : nyeAdresser) { if (a.state == AddressState.CREATE_FAIL) { saveFail.add(a); } else { saveOK.add(a); } } totalNye.addAll(saveOK); totalFejl.addAll(saveFail); totalUkendtDAV.putAll(ukendteDAV); totalDobbeltDav.putAll(dobbeltGadeidDAV); antalBehandlede++; sb.append("

Kommune: " + kommunenavn + "

\n"); sb.append("Kommunekode: " + kommunekode + "
\n"); sb.append("Antal Inserts: " + saveOK.size() + "
\n"); sb.append("Antal insert fails: " + saveFail.size() + "
\n"); sb.append("Antal ukendte dav: " + ukendteDAV.size() + "
\n"); sb.append("Antal dobbelt gadeid dav: " + dobbeltGadeidDAV.size() + "
\n"); } logger.info("Sendingreport mail"); headerSB.append("Antal Inserts: " + totalNye.size() + "
\n"); headerSB.append("Antal insert fails: " + totalFejl.size() + "
\n"); headerSB.append("Antal ukendte dav: " + totalUkendtDAV.size() + "
\n"); headerSB.append("Antal dobbelt gadeid dav: " + totalDobbeltDav.size() + "
\n"); headerSB.append("Elapsed: " + timer.getElapsed() + " ms.
\n"); Map attachments = new TreeMap<>(); attachments.put("saveok.csv", buildAddressFile(totalNye) ); attachments.put("savefailed.csv", buildAddressFile(totalFejl) ); attachments.put("ukendte_dav.csv", buildDavFile(totalUkendtDAV) ); attachments.put("dobbelt_dav_gadeid.csv", buildDavFile(totalDobbeltDav) ); MailSender.sendMailWithAttachments(subject, headerSB.toString() + sb.toString(), attachments); //Cleaning dedup.clear(); } private String buildDavFile(CountMap dav) { StringBuilder sb = new StringBuilder(); sb.append("DAVid;Vejnavn;Antal\r\n"); for ( Entry entry : dav.entrySet()) { List split = Splitter.on(' ').limit(2).splitToList(entry.getKey()); sb.append( split.get(0) ).append(";");//DAVid sb.append( split.get(1) ).append(";");//vejnavn sb.append( entry.getValue() ).append("\r\n");//vejnavn } return sb.toString(); } private String buildAddressFile(List
addr) { Collections.sort(addr); //brug dens egen compareTo funktion StringBuilder sb = new StringBuilder(); sb.append("GadeID;Postnr;Kom.kode;Vejkode;Vejnavn;Husnr;Litra\r\n"); for (Address a: addr) { sb.append(a.gadeid).append(";"); sb.append(a.postnr).append(";"); sb.append(a.kommunekode).append(";"); sb.append(a.vejkode).append(";"); sb.append(a.vejnavn).append(";"); sb.append(a.husnr).append(";"); sb.append(a.husnrbogstav).append("\r\n"); } return sb.toString(); } String getDAVid(Address addr) { String dav = String.format("%04d%03d%04d", addr.postnr, addr.kommunekode, addr.vejkode); return dedup.getInstance(dav); } @Override public String getDescription() { return "DawaTask"; } @Override public String getDetail() { return "Region: " + regionCode + "/" + regionName; } @Override public double getPercentCompleted() { if (antalKommuner == 0) { return 0; } return (antalBehandlede / ((double)antalKommuner) ) * 100.0; } }