adressevedligehold/dawa/DawaTask.java

package dk.daoas.adressevedligehold.dawa;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.TreeSet;

import dk.daoas.adressevedligehold.MailSender;
import dk.daoas.adressevedligehold.beans.Address;
import dk.daoas.adressevedligehold.beans.Address.AddressState;
import dk.daoas.adressevedligehold.db.DatabaseCoverageUpdate;
import dk.daoas.adressevedligehold.tasks.Task;
import dk.daoas.adressevedligehold.tasks.TaskLogger;
import dk.daoas.adressevedligehold.util.DeduplicateHelper;
import dk.daoas.adressevedligehold.util.TimingHelper;

/*
 * Alle regioner: http://dawa.aws.dk/regioner/
 * Enkelt region: http://dawa.aws.dk/regioner/{region}
 */


public class DawaTask extends Task {
        
        final String kommentarTemplate = "Oprettet via DAWA";
        
        private TaskLogger logger = TaskLogger.getInstance(); 
        
        private short regionCode;
        private String regionName;
        
        
        DatabaseCoverageUpdate db = new DatabaseCoverageUpdate();
        
        DeduplicateHelper<String> dedup = new DeduplicateHelper<>(); 
        
        public DawaTask(short regionCode, String regionName){
                this.regionCode = regionCode;           
                this.regionName = regionName;
                
        }

        @Override
        protected void taskRun() throws Exception {
                TimingHelper timer = new TimingHelper();
                        
                DawaWebservice dawa = new DawaWebservice();
                TreeMap<String, DawaGade> search = new TreeMap<>();
                
                // stage 1 - hent alle data                     
                List<Address> adresser = db.getAllAdresses();
                
                
                // stage 2 opbyg søge struktur         
                logger.info("Building search structure");

                
                for (Address addr : adresser) {
                        String dav = getDAVid(addr);
                        DawaGade gade = search.get(dav);
                        if (gade == null) {
                                gade = new DawaGade();
                                search.put(dav, gade);
                        }
                        gade.gadeids.add( addr.gadeid );
                        gade.adresser.add(addr);                        
                }
                logger.info("Build search structure done");
                
                Map<String,Integer> kommuner = dawa.kommunerIRegion(regionCode);
                
                String subject = "DAWA indlæsning done";
                
                StringBuilder headerSB = new StringBuilder();
                headerSB.append("<h1>Indlæsnings rapport for " + regionName + " / " + regionCode + " </h1><br>");
                
                int totalAntalNye = 0;
                int totalAntalFejl = 0;
                int totalAntalUkendt = 0;
                int totalAntalDobbelt = 0;
                
                
                StringBuilder sb = new StringBuilder();
                
                for (Map.Entry<String, Integer> entry : kommuner.entrySet()) {
                        int kommunekode = entry.getValue();
                        String kommunenavn = entry.getKey();
                        
                        logger.info("Behandler: " + kommunekode + " " + kommunenavn);
                
                        
                        List<Address> dawaListe = dawa.adresserIKommune(kommunekode);
                        // stage 3 søg         
                        Set<String> ukendteDAV = new TreeSet<>();
                        Set<String> dobbeltGadeidDAV = new TreeSet<>();
                        List<Address> nyeAdresser = new ArrayList<>();
                        
                        for (Address dawaAddr : dawaListe) {
                                String dav = getDAVid(dawaAddr);
                                DawaGade gade = search.get(dav);
                                if (gade == null) {
                                        ukendteDAV.add(dav + " " + dawaAddr.vejnavn);
                                        continue;
                                }
                                
                                if (gade.gadeids.size() != 1) {
                                        dobbeltGadeidDAV.add(dav + " " + dawaAddr.vejnavn);
                                }
                                
                                boolean fundet = false;
                                for (Address target : gade.adresser) {
                                        if ( target.husnr == dawaAddr.husnr && target.husnrbogstav.equals(dawaAddr.husnrbogstav) ) {
                                                fundet = true;
                                                break;
                                        }       
                                }
                                
                                if (fundet == false) {
                                        dawaAddr.gadeid = gade.gadeids.iterator().next();
                                        dawaAddr.state = AddressState.CREATED;
                                        dawaAddr.kommentar = kommentarTemplate;
                                        nyeAdresser.add(dawaAddr);
                                }                       
                        }
                        logger.info("Address Search done - starting save to DB");
                        
                        db.saveNewAddresses(nyeAdresser);
                        
                        logger.info("Save to db done - sending report mail");
                        
                        List<Address> saveOK = new ArrayList<>();
                        List<Address> saveFail = new ArrayList<>();
                        for (Address a : nyeAdresser) {
                                if (a.state == AddressState.CREATE_FAIL) {
                                        saveFail.add(a);
                                } else {
                                        saveOK.add(a);
                                }
                        }
                        
                        totalAntalNye += saveOK.size();
                        totalAntalFejl += saveFail.size();
                        totalAntalUkendt += ukendteDAV.size();
                        totalAntalDobbelt += dobbeltGadeidDAV.size();
        
                        
                        sb.append("<h2>Kommune: " + kommunenavn + " / " + kommunekode + "</h2><br>\n");
                        sb.append("Antal Inserts: " + saveOK.size() + "<br>\n");
                        sb.append("Antal insert fails: " + saveFail.size() + "<br>\n");
                        sb.append("Antal ukendte dav: " + ukendteDAV.size() + "<br>\n");
                        sb.append("Antal dobbelt gadeid dav: " + dobbeltGadeidDAV.size() + "<br>\n");
                        
                        sb.append("<h3>Adresser ok:</h3>\n");
                        addressTable(sb, saveOK);
                        
                        sb.append("<h3>SQL INSERT fejl:</h3>\n");
                        addressTable(sb, saveFail);
                        
                        sb.append("<h3>Ukendte DAV:</h3>\n");
                        for (String dav : ukendteDAV) {
                                sb.append(dav + "<br>\n");
                        }
                        
                        sb.append("<h3>Dobbelt gadeid DAV:</h3>\n");
                        for (String dav : dobbeltGadeidDAV) {
                                sb.append(dav + "<br>\n");
                        }
                        
                }
                
                headerSB.append("Antal Inserts: " + totalAntalNye + "<br>\n");
                headerSB.append("Antal insert fails: " + totalAntalFejl + "<br>\n");
                headerSB.append("Antal ukendte dav: " + totalAntalUkendt + "<br>\n");
                headerSB.append("Antal dobbelt gadeid dav: " + totalAntalDobbelt + "<br>\n");
                headerSB.append("Elapsed: " + timer.getElapsed() + " ms.<br>\n");
                
                MailSender.sendMail(subject, headerSB.toString() + sb.toString() );

        }
        
        private void addressTable(StringBuilder sb, List<Address> addr) {
                sb.append("<table border=0>\n");
                sb.append("<tr><th>GadeID</th><th>Postnr</th><th>Kom.kode</th><th>Vejkode</th><th>Vejnavn</th><th>Husnr<th><th>Litra</th></tr>");
                for (Address a: addr) {
                        sb.append("<tr>");
                        sb.append("<td>&nbsp;").append(a.gadeid).append(" </td>");
                        sb.append("<td>&nbsp;").append(a.postnr).append(" </td>");
                        sb.append("<td>&nbsp;").append(a.kommunekode).append(" </td>");
                        sb.append("<td>&nbsp;").append(a.vejkode).append(" </td>");
                        sb.append("<td>&nbsp;").append(a.vejnavn).append(" </td>");
                        sb.append("<td>&nbsp;").append(a.husnr).append(" </td>");
                        sb.append("<td>&nbsp;").append(a.husnrbogstav).append(" </td>");
                }
                sb.append("</table><br>\n");
                
        }
        
        String getDAVid(Address addr) {
                String dav = String.format("%04d%03d%04d", addr.postnr, addr.kommunekode, addr.vejkode);
                
                return dedup.getInstance(dav);
        }

        @Override
        public String getDescription() {
                return "DawaTask";
        }

        @Override
        public String getDetail() {
                return "Region: " + regionCode + "/" + regionName;
        }

        @Override
        public double getPercentCompleted() {
                return 0;
        }
        

}
1	package dk.daoas.adressevedligehold.dawa;
2
3	import java.util.ArrayList;
4	import java.util.List;
5	import java.util.Map;
6	import java.util.Set;
7	import java.util.TreeMap;
8	import java.util.TreeSet;
9
10	import dk.daoas.adressevedligehold.MailSender;
11	import dk.daoas.adressevedligehold.beans.Address;
12	import dk.daoas.adressevedligehold.beans.Address.AddressState;
13	import dk.daoas.adressevedligehold.db.DatabaseCoverageUpdate;
14	import dk.daoas.adressevedligehold.tasks.Task;
15	import dk.daoas.adressevedligehold.tasks.TaskLogger;
16	import dk.daoas.adressevedligehold.util.DeduplicateHelper;
17	import dk.daoas.adressevedligehold.util.TimingHelper;
18
19	/*
20	* Alle regioner: http://dawa.aws.dk/regioner/
21	* Enkelt region: http://dawa.aws.dk/regioner/{region}
22	*/
23
24
25	public class DawaTask extends Task {
26
27	final String kommentarTemplate = "Oprettet via DAWA";
28
29	private TaskLogger logger = TaskLogger.getInstance();
30
31	private short regionCode;
32	private String regionName;
33
34
35
36	DatabaseCoverageUpdate db = new DatabaseCoverageUpdate();
37
38	DeduplicateHelper<String> dedup = new DeduplicateHelper<>();
39
40	public DawaTask(short regionCode, String regionName){
41	this.regionCode = regionCode;
42	this.regionName = regionName;
43
44	}
45
46	@Override
47	protected void taskRun() throws Exception {
48	TimingHelper timer = new TimingHelper();
49
50	DawaWebservice dawa = new DawaWebservice();
51	TreeMap<String, DawaGade> search = new TreeMap<>();
52
53	// stage 1 - hent alle data
54	List<Address> adresser = db.getAllAdresses();
55
56
57	// stage 2 opbyg søge struktur
58	logger.info("Building search structure");
59
60
61	for (Address addr : adresser) {
62	String dav = getDAVid(addr);
63	DawaGade gade = search.get(dav);
64	if (gade == null) {
65	gade = new DawaGade();
66	search.put(dav, gade);
67	}
68	gade.gadeids.add( addr.gadeid );
69	gade.adresser.add(addr);
70	}
71	logger.info("Build search structure done");
72
73	Map<String,Integer> kommuner = dawa.kommunerIRegion(regionCode);
74
75	String subject = "DAWA indlæsning done";
76
77	StringBuilder headerSB = new StringBuilder();
78	headerSB.append("<h1>Indlæsnings rapport for " + regionName + " / " + regionCode + " </h1><br>");
79
80	int totalAntalNye = 0;
81	int totalAntalFejl = 0;
82	int totalAntalUkendt = 0;
83	int totalAntalDobbelt = 0;
84
85
86	StringBuilder sb = new StringBuilder();
87
88	for (Map.Entry<String, Integer> entry : kommuner.entrySet()) {
89	int kommunekode = entry.getValue();
90	String kommunenavn = entry.getKey();
91
92	logger.info("Behandler: " + kommunekode + " " + kommunenavn);
93
94
95	List<Address> dawaListe = dawa.adresserIKommune(kommunekode);
96	// stage 3 søg
97	Set<String> ukendteDAV = new TreeSet<>();
98	Set<String> dobbeltGadeidDAV = new TreeSet<>();
99	List<Address> nyeAdresser = new ArrayList<>();
100
101	for (Address dawaAddr : dawaListe) {
102	String dav = getDAVid(dawaAddr);
103	DawaGade gade = search.get(dav);
104	if (gade == null) {
105	ukendteDAV.add(dav + " " + dawaAddr.vejnavn);
106	continue;
107	}
108
109	if (gade.gadeids.size() != 1) {
110	dobbeltGadeidDAV.add(dav + " " + dawaAddr.vejnavn);
111	}
112
113	boolean fundet = false;
114	for (Address target : gade.adresser) {
115	if ( target.husnr == dawaAddr.husnr && target.husnrbogstav.equals(dawaAddr.husnrbogstav) ) {
116	fundet = true;
117	break;
118	}
119	}
120
121	if (fundet == false) {
122	dawaAddr.gadeid = gade.gadeids.iterator().next();
123	dawaAddr.state = AddressState.CREATED;
124	dawaAddr.kommentar = kommentarTemplate;
125	nyeAdresser.add(dawaAddr);
126	}
127	}
128	logger.info("Address Search done - starting save to DB");
129
130	db.saveNewAddresses(nyeAdresser);
131
132	logger.info("Save to db done - sending report mail");
133
134	List<Address> saveOK = new ArrayList<>();
135	List<Address> saveFail = new ArrayList<>();
136	for (Address a : nyeAdresser) {
137	if (a.state == AddressState.CREATE_FAIL) {
138	saveFail.add(a);
139	} else {
140	saveOK.add(a);
141	}
142	}
143
144	totalAntalNye += saveOK.size();
145	totalAntalFejl += saveFail.size();
146	totalAntalUkendt += ukendteDAV.size();
147	totalAntalDobbelt += dobbeltGadeidDAV.size();
148
149
150	sb.append("<h2>Kommune: " + kommunenavn + " / " + kommunekode + "</h2><br>\n");
151	sb.append("Antal Inserts: " + saveOK.size() + "<br>\n");
152	sb.append("Antal insert fails: " + saveFail.size() + "<br>\n");
153	sb.append("Antal ukendte dav: " + ukendteDAV.size() + "<br>\n");
154	sb.append("Antal dobbelt gadeid dav: " + dobbeltGadeidDAV.size() + "<br>\n");
155
156	sb.append("<h3>Adresser ok:</h3>\n");
157	addressTable(sb, saveOK);
158
159	sb.append("<h3>SQL INSERT fejl:</h3>\n");
160	addressTable(sb, saveFail);
161
162	sb.append("<h3>Ukendte DAV:</h3>\n");
163	for (String dav : ukendteDAV) {
164	sb.append(dav + "<br>\n");
165	}
166
167	sb.append("<h3>Dobbelt gadeid DAV:</h3>\n");
168	for (String dav : dobbeltGadeidDAV) {
169	sb.append(dav + "<br>\n");
170	}
171
172	}
173
174	headerSB.append("Antal Inserts: " + totalAntalNye + "<br>\n");
175	headerSB.append("Antal insert fails: " + totalAntalFejl + "<br>\n");
176	headerSB.append("Antal ukendte dav: " + totalAntalUkendt + "<br>\n");
177	headerSB.append("Antal dobbelt gadeid dav: " + totalAntalDobbelt + "<br>\n");
178	headerSB.append("Elapsed: " + timer.getElapsed() + " ms.<br>\n");
179
180	MailSender.sendMail(subject, headerSB.toString() + sb.toString() );
181
182	}
183
184	private void addressTable(StringBuilder sb, List<Address> addr) {
185	sb.append("<table border=0>\n");
186	sb.append("<tr><th>GadeID</th><th>Postnr</th><th>Kom.kode</th><th>Vejkode</th><th>Vejnavn</th><th>Husnr<th><th>Litra</th></tr>");
187	for (Address a: addr) {
188	sb.append("<tr>");
189	sb.append("<td> ").append(a.gadeid).append(" </td>");
190	sb.append("<td> ").append(a.postnr).append(" </td>");
191	sb.append("<td> ").append(a.kommunekode).append(" </td>");
192	sb.append("<td> ").append(a.vejkode).append(" </td>");
193	sb.append("<td> ").append(a.vejnavn).append(" </td>");
194	sb.append("<td> ").append(a.husnr).append(" </td>");
195	sb.append("<td> ").append(a.husnrbogstav).append(" </td>");
196	}
197	sb.append("</table><br>\n");
198
199	}
200
201	String getDAVid(Address addr) {
202	String dav = String.format("%04d%03d%04d", addr.postnr, addr.kommunekode, addr.vejkode);
203
204	return dedup.getInstance(dav);
205	}
206
207	@Override
208	public String getDescription() {
209	return "DawaTask";
210	}
211
212	@Override
213	public String getDetail() {
214	return "Region: " + regionCode + "/" + regionName;
215	}
216
217	@Override
218	public double getPercentCompleted() {
219	return 0;
220	}
221
222
223
224	}