adressevedligehold/dawa/DawaTask.java

package dk.daoas.adressevedligehold.dawa;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.TreeMap;

import com.google.common.base.Splitter;

import dk.daoas.adressevedligehold.MailSender;
import dk.daoas.adressevedligehold.beans.Address;
import dk.daoas.adressevedligehold.beans.Address.AddressState;
import dk.daoas.adressevedligehold.db.DatabaseCoverageUpdate;
import dk.daoas.adressevedligehold.tasks.Task;
import dk.daoas.adressevedligehold.tasks.TaskLogger;
import dk.daoas.adressevedligehold.util.CountMap;
import dk.daoas.adressevedligehold.util.DeduplicateHelper;
import dk.daoas.adressevedligehold.util.TimingHelper;

/*
 * Alle regioner: http://dawa.aws.dk/regioner/
 * Enkelt region: http://dawa.aws.dk/regioner/{region}
 */


public class DawaTask extends Task {
        
        final static String KOMMENTAR_TEMPLATE = "Oprettet via DAWA";
        
        private TaskLogger logger = TaskLogger.getInstance(); 
        
        int antalBehandlede = 0;
        int antalKommuner = 0;
        
        
        String status = "";
        
        DeduplicateHelper<String> dedup = new DeduplicateHelper<>();
        
        
        public DawaTask(){              
        }

        @Override
        protected void taskRun() throws Exception {
                TimingHelper timer = new TimingHelper();
                                
                DatabaseCoverageUpdate db = new DatabaseCoverageUpdate();
                
                DawaWebservice dawa = new DawaWebservice();
                TreeMap<String, DawaGade> search = new TreeMap<>();
                
                // stage 1 - hent alle data
                status = "Henter Data";
                List<Address> adresser = db.getAllAdresses();
                
                
                // stage 2 opbyg søge struktur
                status = "Bygger søgestruktur";
                logger.info("Building search structure");

                
                for (Address addr : adresser) {
                        String dav = getDAVid(addr);
                        DawaGade gade = search.get(dav);
                        if (gade == null) {
                                gade = new DawaGade();
                                search.put(dav, gade);
                        }
                        gade.gadeids.add( addr.gadeid );
                        gade.adresser.add(addr);                        
                }
                logger.info("Build search structure done");
                
                Map<String,Integer> kommuner = dawa.kommuner();
                
                String subject = "DAWA indlæsning";
                
                StringBuilder headerSB = new StringBuilder();
                headerSB.append("<h1>DAWA Indlæsnings rapport</h1><br>");
                
                List<Address> totalNye = new ArrayList<>();
                List<Address> totalFejl = new ArrayList<>();
                CountMap totalUkendtDAV = new CountMap();
                CountMap totalDobbeltDav = new CountMap();
                
                
                antalKommuner = kommuner.size();
                
                
                StringBuilder sb = new StringBuilder();
                
                for (Map.Entry<String, Integer> entry : kommuner.entrySet()) {
                        int kommunekode = entry.getValue();
                        String kommunenavn = entry.getKey();
                        
                        
                        status = "Behandler: " + kommunenavn;
                        logger.info("Behandler: " + kommunekode + " " + kommunenavn);
                
                        
                        List<Address> dawaListe = dawa.adresserIKommune(kommunekode);
                        // stage 3 søg         
                        CountMap ukendteDAV = new CountMap();
                        CountMap dobbeltGadeidDAV = new CountMap();
                        List<Address> nyeAdresser = new ArrayList<>();
                        
                        for (Address dawaAddr : dawaListe) {
                                String dav = getDAVid(dawaAddr);
                                DawaGade gade = search.get(dav);
                                if (gade == null) {
                                        String key = dav + " " + dawaAddr.vejnavn;
                                        ukendteDAV.increment(key);
                                        continue;
                                }
                                
                                if (gade.gadeids.size() != 1) {
                                        String key = dav + " " + dawaAddr.vejnavn;
                                        dobbeltGadeidDAV.increment(key);
                                        continue;
                                }
                                
                                boolean fundet = false;
                                for (Address target : gade.adresser) {
                                        if ( target.husnr == dawaAddr.husnr && target.husnrbogstav.equals(dawaAddr.husnrbogstav) ) {
                                                fundet = true;
                                                break;
                                        }       
                                }
                                
                                if (fundet == false) {
                                        dawaAddr.gadeid = gade.gadeids.iterator().next();
                                        dawaAddr.state = AddressState.CREATED;
                                        dawaAddr.kommentar = KOMMENTAR_TEMPLATE;
                                        nyeAdresser.add(dawaAddr);
                                }                       
                        }
                        logger.info("Address Search done - starting save to DB");
                        
                        db.saveNewAddresses(nyeAdresser);
                        
                        logger.info("Save to db done - adding to report mail");
                        
                        List<Address> saveOK = new ArrayList<>();
                        List<Address> saveFail = new ArrayList<>();
                        for (Address a : nyeAdresser) {
                                if (a.state == AddressState.CREATE_FAIL) {
                                        saveFail.add(a);
                                } else {
                                        saveOK.add(a);
                                }
                        }
                        
                        totalNye.addAll(saveOK);
                        totalFejl.addAll(saveFail);
                        
                        totalUkendtDAV.putAll(ukendteDAV);
                        totalDobbeltDav.putAll(dobbeltGadeidDAV);
                
                        
                        antalBehandlede++;
        
                        
                        sb.append("<h2>Kommune: " + kommunenavn + "</h2>\n");
                        
                        sb.append("Kommunekode: " + kommunekode + "<br>\n");
                        sb.append("Antal Inserts: " + saveOK.size() + "<br>\n");
                        sb.append("Antal insert fails: " + saveFail.size() + "<br>\n");
                        sb.append("Antal ukendte dav: " + ukendteDAV.size() + "<br>\n");
                        sb.append("Antal dobbelt gadeid dav: " + dobbeltGadeidDAV.size() + "<br>\n");
                }
                
                logger.info("Sendingreport mail");
                
                headerSB.append("Antal Inserts: " + totalNye.size() + "<br>\n");
                headerSB.append("Antal insert fails: " + totalFejl.size() + "<br>\n");
                headerSB.append("Antal ukendte dav: " + totalUkendtDAV.size() + "<br>\n");
                headerSB.append("Antal dobbelt gadeid dav: " + totalDobbeltDav.size() + "<br>\n");
                headerSB.append("Elapsed: " + timer.getElapsed() + " ms.<br>\n");
                
                Map<String,String> attachments = new TreeMap<>();
                attachments.put("saveok.csv", buildAddressFile(totalNye) );
                attachments.put("savefailed.csv", buildAddressFile(totalFejl) );
                
                attachments.put("ukendte_dav.csv", buildDavFile(totalUkendtDAV) );
                attachments.put("dobbelt_dav_gadeid.csv", buildDavFile(totalDobbeltDav) );
                
                
                MailSender.sendMailWithAttachments(subject, headerSB.toString() + sb.toString(), attachments);

                
                //Cleaning
                dedup.clear();
                status = "";
        }
        
        private String buildDavFile(CountMap dav) {
                StringBuilder sb = new StringBuilder();
                
                sb.append("DAVid;Vejnavn;Antal\r\n");
                for ( Entry<String,Integer> entry : dav.entrySet()) {
                        
                        List<String> split = Splitter.on(' ').limit(2).splitToList(entry.getKey());
                        sb.append( split.get(0) ).append(";");//DAVid
                        sb.append( split.get(1) ).append(";");//vejnavn
                        
                        sb.append( entry.getValue() ).append("\r\n");//vejnavn
                        
                }
                
                return sb.toString();
        }
        
        private String buildAddressFile(List<Address> addr) {
                
                Collections.sort(addr); //brug dens egen compareTo funktion
                
                StringBuilder sb = new StringBuilder();
                                
                sb.append("GadeID;Postnr;Kom.kode;Vejkode;Vejnavn;Husnr;Litra\r\n");
                for (Address a: addr) {
                        sb.append(a.gadeid).append(";");
                        sb.append(a.postnr).append(";");
                        sb.append(a.kommunekode).append(";");
                        sb.append(a.vejkode).append(";");
                        sb.append(a.vejnavn).append(";");
                        sb.append(a.husnr).append(";");
                        sb.append(a.husnrbogstav).append("\r\n");
                }
                return sb.toString();
                
        }
        
        
        String getDAVid(Address addr) {
                String dav = String.format("%04d%03d%04d", addr.postnr, addr.kommunekode, addr.vejkode);
                
                return dedup.getInstance(dav);
        }

        @Override
        public String getDescription() {
                return "DawaTask";
        }

        @Override
        public String getDetail() {
                return status;
        }

        @Override
        public double getPercentCompleted() {
                if (antalKommuner == 0) {
                        return 0;       
                }
                
                return  (antalBehandlede / ((double)antalKommuner) ) * 100.0;
                
        }
        

}
1	torben	3002	package dk.daoas.adressevedligehold.dawa;
2
3	torben	3003	import java.util.ArrayList;
4	torben	3069	import java.util.Collections;
5	torben	3003	import java.util.List;
6	torben	3006	import java.util.Map;
7	torben	3069	import java.util.Map.Entry;
8	torben	3003	import java.util.TreeMap;
9	torben	3002
10	torben	3069	import com.google.common.base.Splitter;
11
12	torben	3003	import dk.daoas.adressevedligehold.MailSender;
13			import dk.daoas.adressevedligehold.beans.Address;
14			import dk.daoas.adressevedligehold.beans.Address.AddressState;
15			import dk.daoas.adressevedligehold.db.DatabaseCoverageUpdate;
16	torben	3002	import dk.daoas.adressevedligehold.tasks.Task;
17	torben	3003	import dk.daoas.adressevedligehold.tasks.TaskLogger;
18	torben	3012	import dk.daoas.adressevedligehold.util.CountMap;
19	torben	3003	import dk.daoas.adressevedligehold.util.DeduplicateHelper;
20	torben	3006	import dk.daoas.adressevedligehold.util.TimingHelper;
21	torben	3002
22			/*
23			* Alle regioner: http://dawa.aws.dk/regioner/
24			* Enkelt region: http://dawa.aws.dk/regioner/{region}
25			*/
26
27
28			public class DawaTask extends Task {
29
30	torben	3021	final static String KOMMENTAR_TEMPLATE = "Oprettet via DAWA";
31	torben	3003
32			private TaskLogger logger = TaskLogger.getInstance();
33
34	torben	3029	int antalBehandlede = 0;
35			int antalKommuner = 0;
36	torben	3002
37	torben	3003
38	torben	3072	String status = "";
39
40	torben	3034	DeduplicateHelper<String> dedup = new DeduplicateHelper<>();
41	torben	3029
42	torben	3003
43
44	torben	3072	public DawaTask(){
45	torben	3002	}
46
47			@Override
48			protected void taskRun() throws Exception {
49	torben	3006	TimingHelper timer = new TimingHelper();
50	torben	3034
51			DatabaseCoverageUpdate db = new DatabaseCoverageUpdate();
52
53	torben	3003	DawaWebservice dawa = new DawaWebservice();
54	torben	3006	TreeMap<String, DawaGade> search = new TreeMap<>();
55	torben	3003
56	torben	3072	// stage 1 - hent alle data
57			status = "Henter Data";
58	torben	3003	List<Address> adresser = db.getAllAdresses();
59
60
61	torben	3072	// stage 2 opbyg søge struktur
62			status = "Bygger søgestruktur";
63	torben	3003	logger.info("Building search structure");
64	torben	3006
65	torben	3003
66			for (Address addr : adresser) {
67			String dav = getDAVid(addr);
68			DawaGade gade = search.get(dav);
69			if (gade == null) {
70			gade = new DawaGade();
71			search.put(dav, gade);
72			}
73			gade.gadeids.add( addr.gadeid );
74			gade.adresser.add(addr);
75			}
76			logger.info("Build search structure done");
77
78	torben	3072	Map<String,Integer> kommuner = dawa.kommuner();
79	torben	3003
80	torben	3072	String subject = "DAWA indlæsning";
81	torben	3006
82			StringBuilder headerSB = new StringBuilder();
83	torben	3072	headerSB.append("<h1>DAWA Indlæsnings rapport</h1><br>");
84	torben	3006
85	torben	3069	List<Address> totalNye = new ArrayList<>();
86			List<Address> totalFejl = new ArrayList<>();
87			CountMap totalUkendtDAV = new CountMap();
88			CountMap totalDobbeltDav = new CountMap();
89	torben	3006
90	torben	3069
91
92	torben	3029	antalKommuner = kommuner.size();
93	torben	3006
94	torben	3029
95	torben	3006	StringBuilder sb = new StringBuilder();
96
97			for (Map.Entry<String, Integer> entry : kommuner.entrySet()) {
98			int kommunekode = entry.getValue();
99			String kommunenavn = entry.getKey();
100
101	torben	3072
102			status = "Behandler: " + kommunenavn;
103	torben	3006	logger.info("Behandler: " + kommunekode + " " + kommunenavn);
104
105
106			List<Address> dawaListe = dawa.adresserIKommune(kommunekode);
107			// stage 3 søg
108	torben	3012	CountMap ukendteDAV = new CountMap();
109			CountMap dobbeltGadeidDAV = new CountMap();
110	torben	3006	List<Address> nyeAdresser = new ArrayList<>();
111
112			for (Address dawaAddr : dawaListe) {
113			String dav = getDAVid(dawaAddr);
114			DawaGade gade = search.get(dav);
115			if (gade == null) {
116	torben	3012	String key = dav + " " + dawaAddr.vejnavn;
117			ukendteDAV.increment(key);
118	torben	3006	continue;
119			}
120
121			if (gade.gadeids.size() != 1) {
122	torben	3012	String key = dav + " " + dawaAddr.vejnavn;
123			dobbeltGadeidDAV.increment(key);
124			continue;
125	torben	3006	}
126
127			boolean fundet = false;
128			for (Address target : gade.adresser) {
129			if ( target.husnr == dawaAddr.husnr && target.husnrbogstav.equals(dawaAddr.husnrbogstav) ) {
130			fundet = true;
131			break;
132			}
133			}
134
135			if (fundet == false) {
136			dawaAddr.gadeid = gade.gadeids.iterator().next();
137			dawaAddr.state = AddressState.CREATED;
138	torben	3021	dawaAddr.kommentar = KOMMENTAR_TEMPLATE;
139	torben	3006	nyeAdresser.add(dawaAddr);
140			}
141	torben	3003	}
142	torben	3006	logger.info("Address Search done - starting save to DB");
143	torben	3003
144	torben	3006	db.saveNewAddresses(nyeAdresser);
145
146	torben	3013	logger.info("Save to db done - adding to report mail");
147	torben	3006
148			List<Address> saveOK = new ArrayList<>();
149			List<Address> saveFail = new ArrayList<>();
150			for (Address a : nyeAdresser) {
151			if (a.state == AddressState.CREATE_FAIL) {
152			saveFail.add(a);
153			} else {
154			saveOK.add(a);
155			}
156	torben	3003	}
157
158	torben	3069	totalNye.addAll(saveOK);
159			totalFejl.addAll(saveFail);
160	torben	3029
161	torben	3069	totalUkendtDAV.putAll(ukendteDAV);
162			totalDobbeltDav.putAll(dobbeltGadeidDAV);
163
164	torben	3029
165			antalBehandlede++;
166	torben	3006
167
168	torben	3008	sb.append("<h2>Kommune: " + kommunenavn + "</h2>\n");
169	torben	3009
170			sb.append("Kommunekode: " + kommunekode + "<br>\n");
171	torben	3006	sb.append("Antal Inserts: " + saveOK.size() + "<br>\n");
172			sb.append("Antal insert fails: " + saveFail.size() + "<br>\n");
173			sb.append("Antal ukendte dav: " + ukendteDAV.size() + "<br>\n");
174			sb.append("Antal dobbelt gadeid dav: " + dobbeltGadeidDAV.size() + "<br>\n");
175	torben	3003	}
176	torben	3069
177	torben	3013	logger.info("Sendingreport mail");
178	torben	3003
179	torben	3069	headerSB.append("Antal Inserts: " + totalNye.size() + "<br>\n");
180			headerSB.append("Antal insert fails: " + totalFejl.size() + "<br>\n");
181			headerSB.append("Antal ukendte dav: " + totalUkendtDAV.size() + "<br>\n");
182			headerSB.append("Antal dobbelt gadeid dav: " + totalDobbeltDav.size() + "<br>\n");
183	torben	3006	headerSB.append("Elapsed: " + timer.getElapsed() + " ms.<br>\n");
184	torben	3003
185	torben	3069	Map<String,String> attachments = new TreeMap<>();
186			attachments.put("saveok.csv", buildAddressFile(totalNye) );
187			attachments.put("savefailed.csv", buildAddressFile(totalFejl) );
188
189			attachments.put("ukendte_dav.csv", buildDavFile(totalUkendtDAV) );
190			attachments.put("dobbelt_dav_gadeid.csv", buildDavFile(totalDobbeltDav) );
191
192
193			MailSender.sendMailWithAttachments(subject, headerSB.toString() + sb.toString(), attachments);
194	torben	3002
195	torben	3034
196			//Cleaning
197			dedup.clear();
198	torben	3072	status = "";
199	torben	3002	}
200	torben	3003
201	torben	3069	private String buildDavFile(CountMap dav) {
202			StringBuilder sb = new StringBuilder();
203
204	torben	3070	sb.append("DAVid;Vejnavn;Antal\r\n");
205	torben	3069	for ( Entry<String,Integer> entry : dav.entrySet()) {
206
207			List<String> split = Splitter.on(' ').limit(2).splitToList(entry.getKey());
208	torben	3070	sb.append( split.get(0) ).append(";");//DAVid
209	torben	3069	sb.append( split.get(1) ).append(";");//vejnavn
210
211			sb.append( entry.getValue() ).append("\r\n");//vejnavn
212
213			}
214
215			return sb.toString();
216			}
217
218			private String buildAddressFile(List<Address> addr) {
219
220			Collections.sort(addr); //brug dens egen compareTo funktion
221
222			StringBuilder sb = new StringBuilder();
223
224			sb.append("GadeID;Postnr;Kom.kode;Vejkode;Vejnavn;Husnr;Litra\r\n");
225	torben	3005	for (Address a: addr) {
226	torben	3069	sb.append(a.gadeid).append(";");
227			sb.append(a.postnr).append(";");
228			sb.append(a.kommunekode).append(";");
229			sb.append(a.vejkode).append(";");
230			sb.append(a.vejnavn).append(";");
231			sb.append(a.husnr).append(";");
232			sb.append(a.husnrbogstav).append("\r\n");
233	torben	3005	}
234	torben	3069	return sb.toString();
235	torben	3005
236			}
237
238	torben	3069
239	torben	3003	String getDAVid(Address addr) {
240			String dav = String.format("%04d%03d%04d", addr.postnr, addr.kommunekode, addr.vejkode);
241
242			return dedup.getInstance(dav);
243			}
244	torben	3002
245			@Override
246			public String getDescription() {
247			return "DawaTask";
248			}
249
250			@Override
251			public String getDetail() {
252	torben	3072	return status;
253	torben	3002	}
254
255			@Override
256			public double getPercentCompleted() {
257	torben	3029	if (antalKommuner == 0) {
258			return 0;
259			}
260
261			return (antalBehandlede / ((double)antalKommuner) ) * 100.0;
262
263	torben	3002	}
264
265
266
267			}