adressevedligehold/dawa/DawaTask.java

package dk.daoas.adressevedligehold.dawa;

import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;

import dk.daoas.adressevedligehold.MailSender;
import dk.daoas.adressevedligehold.beans.Address;
import dk.daoas.adressevedligehold.beans.Address.AddressState;
import dk.daoas.adressevedligehold.db.DatabaseCoverageUpdate;
import dk.daoas.adressevedligehold.tasks.Task;
import dk.daoas.adressevedligehold.tasks.TaskLogger;
import dk.daoas.adressevedligehold.util.CountMap;
import dk.daoas.adressevedligehold.util.DeduplicateHelper;
import dk.daoas.adressevedligehold.util.TimingHelper;

/*
 * Alle regioner: http://dawa.aws.dk/regioner/
 * Enkelt region: http://dawa.aws.dk/regioner/{region}
 */


public class DawaTask extends Task {
        
        final static String KOMMENTAR_TEMPLATE = "Oprettet via DAWA";
        
        private TaskLogger logger = TaskLogger.getInstance(); 
        
        private short regionCode;
        private String regionName;
        
        int antalBehandlede = 0;
        int antalKommuner = 0;
        
        
        DeduplicateHelper<String> dedup = new DeduplicateHelper<>();
        
        
        public DawaTask(short regionCode, String regionName){
                this.regionCode = regionCode;           
                this.regionName = regionName;
                
        }

        @Override
        protected void taskRun() throws Exception {
                TimingHelper timer = new TimingHelper();
                                
                DatabaseCoverageUpdate db = new DatabaseCoverageUpdate();
                
                DawaWebservice dawa = new DawaWebservice();
                TreeMap<String, DawaGade> search = new TreeMap<>();
                
                // stage 1 - hent alle data                     
                List<Address> adresser = db.getAllAdresses();
                
                
                // stage 2 opbyg søge struktur         
                logger.info("Building search structure");

                
                for (Address addr : adresser) {
                        String dav = getDAVid(addr);
                        DawaGade gade = search.get(dav);
                        if (gade == null) {
                                gade = new DawaGade();
                                search.put(dav, gade);
                        }
                        gade.gadeids.add( addr.gadeid );
                        gade.adresser.add(addr);                        
                }
                logger.info("Build search structure done");
                
                Map<String,Integer> kommuner = dawa.kommunerIRegion(regionCode);
                
                String subject = "DAWA indlæsning - " + regionName;
                
                StringBuilder headerSB = new StringBuilder();
                headerSB.append("<h1>Indlæsnings rapport for " + regionName + " </h1><br>");
                
                int totalAntalNye = 0;
                int totalAntalFejl = 0;
                int totalAntalUkendt = 0;
                int totalAntalDobbelt = 0;
                
                antalKommuner = kommuner.size();
                
                
                StringBuilder sb = new StringBuilder();
                
                for (Map.Entry<String, Integer> entry : kommuner.entrySet()) {
                        int kommunekode = entry.getValue();
                        String kommunenavn = entry.getKey();
                        
                        logger.info("Behandler: " + kommunekode + " " + kommunenavn);
                
                        
                        List<Address> dawaListe = dawa.adresserIKommune(kommunekode);
                        // stage 3 søg         
                        CountMap ukendteDAV = new CountMap();
                        CountMap dobbeltGadeidDAV = new CountMap();
                        List<Address> nyeAdresser = new ArrayList<>();
                        
                        for (Address dawaAddr : dawaListe) {
                                String dav = getDAVid(dawaAddr);
                                DawaGade gade = search.get(dav);
                                if (gade == null) {
                                        String key = dav + " " + dawaAddr.vejnavn;
                                        ukendteDAV.increment(key);
                                        continue;
                                }
                                
                                if (gade.gadeids.size() != 1) {
                                        String key = dav + " " + dawaAddr.vejnavn;
                                        dobbeltGadeidDAV.increment(key);
                                        continue;
                                }
                                
                                boolean fundet = false;
                                for (Address target : gade.adresser) {
                                        if ( target.husnr == dawaAddr.husnr && target.husnrbogstav.equals(dawaAddr.husnrbogstav) ) {
                                                fundet = true;
                                                break;
                                        }       
                                }
                                
                                if (fundet == false) {
                                        dawaAddr.gadeid = gade.gadeids.iterator().next();
                                        dawaAddr.state = AddressState.CREATED;
                                        dawaAddr.kommentar = KOMMENTAR_TEMPLATE;
                                        nyeAdresser.add(dawaAddr);
                                }                       
                        }
                        logger.info("Address Search done - starting save to DB");
                        
                        db.saveNewAddresses(nyeAdresser);
                        
                        logger.info("Save to db done - adding to report mail");
                        
                        List<Address> saveOK = new ArrayList<>();
                        List<Address> saveFail = new ArrayList<>();
                        for (Address a : nyeAdresser) {
                                if (a.state == AddressState.CREATE_FAIL) {
                                        saveFail.add(a);
                                } else {
                                        saveOK.add(a);
                                }
                        }
                        
                        totalAntalNye += saveOK.size();
                        totalAntalFejl += saveFail.size();
                        totalAntalUkendt += ukendteDAV.size();
                        totalAntalDobbelt += dobbeltGadeidDAV.size();
                        
                        
                        antalBehandlede++;
        
                        
                        sb.append("<h2>Kommune: " + kommunenavn + "</h2>\n");
                        
                        sb.append("Kommunekode: " + kommunekode + "<br>\n");
                        sb.append("Antal Inserts: " + saveOK.size() + "<br>\n");
                        sb.append("Antal insert fails: " + saveFail.size() + "<br>\n");
                        sb.append("Antal ukendte dav: " + ukendteDAV.size() + "<br>\n");
                        sb.append("Antal dobbelt gadeid dav: " + dobbeltGadeidDAV.size() + "<br>\n");
                        
                        sb.append("<h3>Adresser ok:</h3>\n");
                        addressTable(sb, saveOK);
                        
                        sb.append("<h3>SQL INSERT fejl:</h3>\n");
                        addressTable(sb, saveFail);
                        
                        sb.append("<h3>Ukendte DAV:</h3>\n");
                        for (Map.Entry<String, Integer> dav : ukendteDAV.entrySet()) {                          
                                sb.append(dav.getKey() + " : " + dav.getValue() + "<br>\n");
                        }
                        
                        sb.append("<h3>Dobbelt gadeid DAV:</h3>\n");
                        for (Map.Entry<String, Integer> dav : dobbeltGadeidDAV.entrySet()) {                            
                                sb.append(dav.getKey() + " : " + dav.getValue() + "<br>\n");
                        }                                               
                }
                logger.info("Sendingreport mail");
                
                headerSB.append("Antal Inserts: " + totalAntalNye + "<br>\n");
                headerSB.append("Antal insert fails: " + totalAntalFejl + "<br>\n");
                headerSB.append("Antal ukendte dav: " + totalAntalUkendt + "<br>\n");
                headerSB.append("Antal dobbelt gadeid dav: " + totalAntalDobbelt + "<br>\n");
                headerSB.append("Elapsed: " + timer.getElapsed() + " ms.<br>\n");
                
                MailSender.sendMail(subject, headerSB.toString() + sb.toString() );

                
                //Cleaning
                dedup.clear();
        }
        
        private void addressTable(StringBuilder sb, List<Address> addr) {
                sb.append("<table border=0>\n");
                sb.append("<tr><th>GadeID</th><th>Postnr</th><th>Kom.kode</th><th>Vejkode</th><th>Vejnavn</th><th>Husnr<th><th>Litra</th></tr>");
                for (Address a: addr) {
                        sb.append("<tr>");
                        sb.append("<td>&nbsp;").append(a.gadeid).append(" </td>");
                        sb.append("<td>&nbsp;").append(a.postnr).append(" </td>");
                        sb.append("<td>&nbsp;").append(a.kommunekode).append(" </td>");
                        sb.append("<td>&nbsp;").append(a.vejkode).append(" </td>");
                        sb.append("<td>&nbsp;").append(a.vejnavn).append(" </td>");
                        sb.append("<td>&nbsp;").append(a.husnr).append(" </td>");
                        sb.append("<td>&nbsp;").append(a.husnrbogstav).append(" </td>");
                }
                sb.append("</table><br>\n");
                
        }
        
        String getDAVid(Address addr) {
                String dav = String.format("%04d%03d%04d", addr.postnr, addr.kommunekode, addr.vejkode);
                
                return dedup.getInstance(dav);
        }

        @Override
        public String getDescription() {
                return "DawaTask";
        }

        @Override
        public String getDetail() {
                return "Region: " + regionCode + "/" + regionName;
        }

        @Override
        public double getPercentCompleted() {
                if (antalKommuner == 0) {
                        return 0;       
                }
                
                return  (antalBehandlede / ((double)antalKommuner) ) * 100.0;
                
        }
        

}
1	torben	3002	package dk.daoas.adressevedligehold.dawa;
2
3	torben	3003	import java.util.ArrayList;
4			import java.util.List;
5	torben	3006	import java.util.Map;
6	torben	3003	import java.util.TreeMap;
7	torben	3002
8	torben	3003	import dk.daoas.adressevedligehold.MailSender;
9			import dk.daoas.adressevedligehold.beans.Address;
10			import dk.daoas.adressevedligehold.beans.Address.AddressState;
11			import dk.daoas.adressevedligehold.db.DatabaseCoverageUpdate;
12	torben	3002	import dk.daoas.adressevedligehold.tasks.Task;
13	torben	3003	import dk.daoas.adressevedligehold.tasks.TaskLogger;
14	torben	3012	import dk.daoas.adressevedligehold.util.CountMap;
15	torben	3003	import dk.daoas.adressevedligehold.util.DeduplicateHelper;
16	torben	3006	import dk.daoas.adressevedligehold.util.TimingHelper;
17	torben	3002
18			/*
19			* Alle regioner: http://dawa.aws.dk/regioner/
20			* Enkelt region: http://dawa.aws.dk/regioner/{region}
21			*/
22
23
24			public class DawaTask extends Task {
25
26	torben	3021	final static String KOMMENTAR_TEMPLATE = "Oprettet via DAWA";
27	torben	3003
28			private TaskLogger logger = TaskLogger.getInstance();
29
30	torben	3002	private short regionCode;
31			private String regionName;
32
33	torben	3029	int antalBehandlede = 0;
34			int antalKommuner = 0;
35	torben	3002
36	torben	3003
37	torben	3034	DeduplicateHelper<String> dedup = new DeduplicateHelper<>();
38	torben	3029
39	torben	3003
40	torben	3034
41	torben	3003
42	torben	3002	public DawaTask(short regionCode, String regionName){
43			this.regionCode = regionCode;
44			this.regionName = regionName;
45
46			}
47
48			@Override
49			protected void taskRun() throws Exception {
50	torben	3006	TimingHelper timer = new TimingHelper();
51	torben	3034
52			DatabaseCoverageUpdate db = new DatabaseCoverageUpdate();
53
54	torben	3003	DawaWebservice dawa = new DawaWebservice();
55	torben	3006	TreeMap<String, DawaGade> search = new TreeMap<>();
56	torben	3003
57	torben	3006	// stage 1 - hent alle data
58	torben	3003	List<Address> adresser = db.getAllAdresses();
59
60
61			// stage 2 opbyg søge struktur
62			logger.info("Building search structure");
63	torben	3006
64	torben	3003
65			for (Address addr : adresser) {
66			String dav = getDAVid(addr);
67			DawaGade gade = search.get(dav);
68			if (gade == null) {
69			gade = new DawaGade();
70			search.put(dav, gade);
71			}
72			gade.gadeids.add( addr.gadeid );
73			gade.adresser.add(addr);
74			}
75			logger.info("Build search structure done");
76
77	torben	3006	Map<String,Integer> kommuner = dawa.kommunerIRegion(regionCode);
78	torben	3003
79	torben	3008	String subject = "DAWA indlæsning - " + regionName;
80	torben	3006
81			StringBuilder headerSB = new StringBuilder();
82	torben	3008	headerSB.append("<h1>Indlæsnings rapport for " + regionName + " </h1><br>");
83	torben	3006
84			int totalAntalNye = 0;
85			int totalAntalFejl = 0;
86			int totalAntalUkendt = 0;
87			int totalAntalDobbelt = 0;
88
89	torben	3029	antalKommuner = kommuner.size();
90	torben	3006
91	torben	3029
92	torben	3006	StringBuilder sb = new StringBuilder();
93
94			for (Map.Entry<String, Integer> entry : kommuner.entrySet()) {
95			int kommunekode = entry.getValue();
96			String kommunenavn = entry.getKey();
97
98			logger.info("Behandler: " + kommunekode + " " + kommunenavn);
99
100
101			List<Address> dawaListe = dawa.adresserIKommune(kommunekode);
102			// stage 3 søg
103	torben	3012	CountMap ukendteDAV = new CountMap();
104			CountMap dobbeltGadeidDAV = new CountMap();
105	torben	3006	List<Address> nyeAdresser = new ArrayList<>();
106
107			for (Address dawaAddr : dawaListe) {
108			String dav = getDAVid(dawaAddr);
109			DawaGade gade = search.get(dav);
110			if (gade == null) {
111	torben	3012	String key = dav + " " + dawaAddr.vejnavn;
112			ukendteDAV.increment(key);
113	torben	3006	continue;
114			}
115
116			if (gade.gadeids.size() != 1) {
117	torben	3012	String key = dav + " " + dawaAddr.vejnavn;
118			dobbeltGadeidDAV.increment(key);
119			continue;
120	torben	3006	}
121
122			boolean fundet = false;
123			for (Address target : gade.adresser) {
124			if ( target.husnr == dawaAddr.husnr && target.husnrbogstav.equals(dawaAddr.husnrbogstav) ) {
125			fundet = true;
126			break;
127			}
128			}
129
130			if (fundet == false) {
131			dawaAddr.gadeid = gade.gadeids.iterator().next();
132			dawaAddr.state = AddressState.CREATED;
133	torben	3021	dawaAddr.kommentar = KOMMENTAR_TEMPLATE;
134	torben	3006	nyeAdresser.add(dawaAddr);
135			}
136	torben	3003	}
137	torben	3006	logger.info("Address Search done - starting save to DB");
138	torben	3003
139	torben	3006	db.saveNewAddresses(nyeAdresser);
140
141	torben	3013	logger.info("Save to db done - adding to report mail");
142	torben	3006
143			List<Address> saveOK = new ArrayList<>();
144			List<Address> saveFail = new ArrayList<>();
145			for (Address a : nyeAdresser) {
146			if (a.state == AddressState.CREATE_FAIL) {
147			saveFail.add(a);
148			} else {
149			saveOK.add(a);
150			}
151	torben	3003	}
152
153	torben	3006	totalAntalNye += saveOK.size();
154			totalAntalFejl += saveFail.size();
155			totalAntalUkendt += ukendteDAV.size();
156			totalAntalDobbelt += dobbeltGadeidDAV.size();
157	torben	3029
158
159			antalBehandlede++;
160	torben	3006
161
162	torben	3008	sb.append("<h2>Kommune: " + kommunenavn + "</h2>\n");
163	torben	3009
164			sb.append("Kommunekode: " + kommunekode + "<br>\n");
165	torben	3006	sb.append("Antal Inserts: " + saveOK.size() + "<br>\n");
166			sb.append("Antal insert fails: " + saveFail.size() + "<br>\n");
167			sb.append("Antal ukendte dav: " + ukendteDAV.size() + "<br>\n");
168			sb.append("Antal dobbelt gadeid dav: " + dobbeltGadeidDAV.size() + "<br>\n");
169
170			sb.append("<h3>Adresser ok:</h3>\n");
171			addressTable(sb, saveOK);
172
173			sb.append("<h3>SQL INSERT fejl:</h3>\n");
174			addressTable(sb, saveFail);
175
176			sb.append("<h3>Ukendte DAV:</h3>\n");
177	torben	3012	for (Map.Entry<String, Integer> dav : ukendteDAV.entrySet()) {
178			sb.append(dav.getKey() + " : " + dav.getValue() + "<br>\n");
179	torben	3003	}
180
181	torben	3006	sb.append("<h3>Dobbelt gadeid DAV:</h3>\n");
182	torben	3012	for (Map.Entry<String, Integer> dav : dobbeltGadeidDAV.entrySet()) {
183			sb.append(dav.getKey() + " : " + dav.getValue() + "<br>\n");
184			}
185	torben	3003	}
186	torben	3013	logger.info("Sendingreport mail");
187	torben	3003
188	torben	3006	headerSB.append("Antal Inserts: " + totalAntalNye + "<br>\n");
189			headerSB.append("Antal insert fails: " + totalAntalFejl + "<br>\n");
190			headerSB.append("Antal ukendte dav: " + totalAntalUkendt + "<br>\n");
191			headerSB.append("Antal dobbelt gadeid dav: " + totalAntalDobbelt + "<br>\n");
192			headerSB.append("Elapsed: " + timer.getElapsed() + " ms.<br>\n");
193	torben	3003
194	torben	3006	MailSender.sendMail(subject, headerSB.toString() + sb.toString() );
195	torben	3002
196	torben	3034
197			//Cleaning
198			dedup.clear();
199	torben	3002	}
200	torben	3003
201	torben	3005	private void addressTable(StringBuilder sb, List<Address> addr) {
202			sb.append("<table border=0>\n");
203			sb.append("<tr><th>GadeID</th><th>Postnr</th><th>Kom.kode</th><th>Vejkode</th><th>Vejnavn</th><th>Husnr<th><th>Litra</th></tr>");
204			for (Address a: addr) {
205			sb.append("<tr>");
206			sb.append("<td> ").append(a.gadeid).append(" </td>");
207			sb.append("<td> ").append(a.postnr).append(" </td>");
208			sb.append("<td> ").append(a.kommunekode).append(" </td>");
209			sb.append("<td> ").append(a.vejkode).append(" </td>");
210			sb.append("<td> ").append(a.vejnavn).append(" </td>");
211			sb.append("<td> ").append(a.husnr).append(" </td>");
212			sb.append("<td> ").append(a.husnrbogstav).append(" </td>");
213			}
214			sb.append("</table><br>\n");
215
216			}
217
218	torben	3003	String getDAVid(Address addr) {
219			String dav = String.format("%04d%03d%04d", addr.postnr, addr.kommunekode, addr.vejkode);
220
221			return dedup.getInstance(dav);
222			}
223	torben	3002
224			@Override
225			public String getDescription() {
226			return "DawaTask";
227			}
228
229			@Override
230			public String getDetail() {
231			return "Region: " + regionCode + "/" + regionName;
232			}
233
234			@Override
235			public double getPercentCompleted() {
236	torben	3029	if (antalKommuner == 0) {
237			return 0;
238			}
239
240			return (antalBehandlede / ((double)antalKommuner) ) * 100.0;
241
242	torben	3002	}
243
244
245
246			}