/[projects]/dao/DaoAdresseVedligehold/src/main/java/dk/daoas/adressevedligehold/coveragefileupload/AddressManager.java
ViewVC logotype

Contents of /dao/DaoAdresseVedligehold/src/main/java/dk/daoas/adressevedligehold/coveragefileupload/AddressManager.java

Parent Directory Parent Directory | Revision Log Revision Log


Revision 3024 - (show annotations) (download)
Tue Apr 26 19:34:14 2016 UTC (8 years ago) by torben
File size: 19305 byte(s)
Don't swallow SQL Exceptions in DB layer, but let it propogate up to the task
1 package dk.daoas.adressevedligehold.coveragefileupload;
2
3 import java.sql.SQLException;
4 import java.util.ArrayList;
5 import java.util.Arrays;
6 import java.util.Collections;
7 import java.util.List;
8 import java.util.Map;
9 import java.util.Map.Entry;
10 import java.util.TreeMap;
11
12 import org.apache.commons.lang3.StringUtils;
13
14 import dk.daoas.adressevedligehold.beans.Address;
15 import dk.daoas.adressevedligehold.beans.Address.AddressState;
16 import dk.daoas.adressevedligehold.coveragefileupload.AddressSourceEntry.EntryType;
17 import dk.daoas.adressevedligehold.db.DatabaseCoverageUpdate;
18 import dk.daoas.adressevedligehold.tasks.TaskLogger;
19 import dk.daoas.adressevedligehold.util.DeduplicateHelper;
20 import dk.daoas.adressevedligehold.util.MiscUtils;
21 import dk.daoas.adressevedligehold.util.TimingHelper;
22
23 /*
24 *
25 */
26
27 public class AddressManager {
28
29 private TaskLogger logger = TaskLogger.getInstance();
30
31 private int duplicateCount;
32
33 List<Address> addressList;
34
35 Map<Integer, TreeMap<Short, ArrayList<Address>> > searchStructure;
36
37 ArrayList<AddressSourceEntry> rejectedEntries = new ArrayList<AddressSourceEntry>();
38
39 Map<Integer,String> unknownStreets = new TreeMap<Integer, String>();
40
41 Map<Short,Short> dbkBaneMap = new TreeMap<Short,Short>();
42
43 public AddressManager() throws SQLException {
44 DatabaseCoverageUpdate db = new DatabaseCoverageUpdate();
45
46 searchStructure = new TreeMap<Integer, TreeMap<Short, ArrayList<Address>> >();
47
48 addressList = db.getAllAdresses();
49
50 DeduplicateHelper<Integer> intHelper = new DeduplicateHelper<Integer>();
51 DeduplicateHelper<Short> shortHelper = new DeduplicateHelper<Short>();
52
53 List<ArrayList<Address>> arraylistCache = new ArrayList<ArrayList<Address>>();
54
55 TimingHelper timer = new TimingHelper();
56
57 for (Address a : addressList) {
58 Integer gadeid = intHelper.getInstance( a.gadeid );
59 Short husnr = shortHelper.getInstance( a.husnr );
60 TreeMap<Short, ArrayList<Address>> gade = searchStructure.get( gadeid );
61
62 if (a.dbkBane > 0 && a.postnr<=4999) {
63 Short bane = dbkBaneMap.get(a.postnr);
64 if (bane == null) {
65 dbkBaneMap.put(a.postnr, a.dbkBane);
66 }
67 }
68
69 if (gade == null) {
70 gade = new TreeMap<Short,ArrayList<Address>>();
71 searchStructure.put(gadeid, gade);
72 }
73
74 ArrayList<Address> litraList = gade.get(husnr);
75 if (litraList == null) {
76 litraList = new ArrayList<Address>();
77 gade.put(husnr, litraList);
78 arraylistCache.add(litraList);
79 }
80
81 litraList.add(a);
82 }
83
84 for (ArrayList<Address> list : arraylistCache) {
85 list.trimToSize();
86 }
87
88 logger.info("AddressManager ready, elapsed " + timer.getElapsed() + "ms");
89
90
91 }
92
93 @SuppressWarnings("PMD.CollapsibleIfStatements")
94 public void closeUnvisitedAddresses(String distributor, EntryUgedage ugedage) {
95 for (Address addr : addressList) {
96 if (addr.distributor == null)
97 continue;
98
99 if (addr.distributor.equals(distributor) == false)//irrelevant for denne indlæsning
100 continue;
101
102 int closedCount = 0;
103
104 if (addr.visitedMan == false && ugedage.mandag) {
105 if (addr.ruteMandag != null) {
106 addr.ruteMandag = null;
107 addr.korelisteMandag = null;
108 addr.stateMan = AddressState.CLOSED;
109 closedCount++;
110 }
111 }
112 if (addr.visitedTir == false && ugedage.tirsdag) {
113 if (addr.ruteTirsdag != null) {
114 addr.ruteTirsdag = null;
115 addr.korelisteTirsdag = null;
116 addr.stateTir = AddressState.CLOSED;
117 closedCount++;
118 }
119 }
120
121 if (addr.visitedOns == false && ugedage.onsdag) {
122 if (addr.ruteOnsdag != null) {
123 addr.ruteOnsdag = null;
124 addr.korelisteOnsdag = null;
125 addr.stateOns = AddressState.CLOSED;
126 closedCount++;
127 }
128 }
129
130 if (addr.visitedTor == false && ugedage.torsdag) {
131 if (addr.ruteTorsdag != null) {
132 addr.ruteTorsdag = null;
133 addr.korelisteTorsdag = null;
134 addr.stateTor = AddressState.CLOSED;
135 closedCount++;
136 }
137 }
138
139 if (addr.visitedFre == false && ugedage.fredag) {
140 if (addr.ruteFredag != null) {
141 addr.ruteFredag = null;
142 addr.korelisteFredag = null;
143 addr.stateFre = AddressState.CLOSED;
144 closedCount++;
145 }
146 }
147
148 if (addr.visitedLor == false && ugedage.lordag) {
149 if (addr.ruteLordag != null) {
150 addr.ruteLordag = null;
151 addr.korelisteLordag = null;
152 addr.stateLor = AddressState.CLOSED;
153 closedCount++;
154 }
155 }
156
157 if (addr.visitedSon == false && ugedage.sondag) {
158 if (addr.ruteSondag != null) {
159 addr.ruteSondag = null;
160 addr.korelisteSondag = null;
161 addr.stateSon = AddressState.CLOSED;
162 closedCount++;
163 }
164 }
165
166 if (addr.state == AddressState.NOT_CHANGED && closedCount>0) {
167 addr.state = AddressState.CLOSED;
168 }
169
170 }
171 }
172 public void dumpChanged() {
173 int count = 0;
174 for (Address addr : addressList) {
175 if (addr.state == AddressState.NOT_CHANGED) {
176 continue;
177 }
178 /*if (addr.stateMan == AddressState.NOT_CHANGED || addr.stateMan == AddressState.CREATED) {
179 continue;
180 }*/
181
182 if (addr.stateMan != AddressState.MODIFIED ) {
183 continue;
184 }
185
186 logger.info(addr.state + ": " + addr.toStringExtended() );
187 logger.info(addr.getRuteString() );
188
189 if (count++ > 50)
190 return;
191 }
192 }
193
194 public void writeBackChanges() throws Exception{
195 logger.info("Writing back changes");
196 TimingHelper timer = new TimingHelper();
197
198 List<Address> updates = new ArrayList<Address>(1024*1024);
199 List<Address> inserts = new ArrayList<Address>(16*1024);
200
201 for (Address addr : addressList) {
202 if (addr.state == AddressState.CREATED) {
203 inserts.add(addr);
204 }
205 if (addr.state == AddressState.MODIFIED || addr.state == AddressState.OPENED || addr.state == AddressState.CLOSED) {
206 updates.add(addr);
207 }
208 }
209
210 DatabaseCoverageUpdate db = new DatabaseCoverageUpdate();
211
212 db.updateAddresses(updates);
213 db.saveNewAddresses(inserts);
214
215
216
217 logger.info("Writeback done: elapsed " + timer.getElapsed() + "ms.");
218 }
219
220
221 public void visit(AddressSourceEntry entry) {
222 if (entry.type == EntryType.TypeSingleAddress) {
223 visitSingle(entry);
224 } else {
225 visitRange(entry);
226 }
227 }
228
229 private void visitSingle(AddressSourceEntry entry) {
230 TreeMap<Short, ArrayList<Address>> gade = searchStructure.get( entry.gadeid );
231 if (gade == null) {
232 /* No need to register unknown streets, since we create all entries that we can
233 if (! unknownStreets.containsKey(entry.gadeid)) {
234 unknownStreets.put(entry.gadeid, entry.vejnavn);
235 }*/
236
237 createFromEntry(entry); // if we get here there was no match - so we need to create it
238 return;
239 }
240
241 ArrayList<Address> litraList = gade.get(entry.husnr);
242 if (litraList == null) {
243 createFromEntry(entry); // if we get here there was no match - so we need to create it
244 return;
245 }
246 boolean found = false;
247 for (Address addr : litraList) {
248 if (addr.husnrbogstav.equals(entry.litra) ) {
249 updateAddress(addr, entry);
250 found = true; // 1 visit should be enough but as long as there's duplicates on gadeid+husnr+litra we will visit them all
251 //when the issue with duplicates is resolved this should be reverted to a return or break
252 }
253 }
254 if (found == false) {
255 createFromEntry(entry); // if we get here there was no match - so we need to create it
256 }
257 }
258
259 @SuppressWarnings("PMD.CollapsibleIfStatements")
260 private void visitRange(AddressSourceEntry entry) {
261
262 TreeMap<Short, ArrayList<Address>> gade = searchStructure.get( entry.gadeid );
263 if (gade == null) {
264 if (! unknownStreets.containsKey(entry.gadeid)) {
265 unknownStreets.put(entry.gadeid, entry.vejnavn);
266 }
267
268 logger.info("[Range] Ukendt gadeID " + entry);
269 return;
270 }
271
272 for (Entry<Short, ArrayList<Address>> husnrEntry: gade.entrySet()) {
273
274 short husnummer = husnrEntry.getKey();
275
276 if ( (entry.husnr % 2) != (husnummer%2) ){ //lige/ulige skal passe sammen
277 continue;
278 }
279
280 if ( husnummer < entry.husnr || husnummer > entry.tilHusnr) {
281 continue;
282 }
283
284 ArrayList<Address> litraList = husnrEntry.getValue();
285
286 /* a=adressetabel u=input
287 * and (a.husnr>u.FraHusNr OR (a.husnr=u.FraHusNr AND a.HusnrBogstav >= u.FraBog))
288 * AND (a.husnr<u.TilHusNr OR (a.husnr=u.TilHusNr AND a.HusnrBogstav <= u.TilBog))
289 */
290
291
292 for (Address addr: litraList) {
293 if (addr.husnr > entry.husnr || (addr.husnr == entry.husnr && addr.husnrbogstav.compareTo(entry.litra) >= 0 )) {
294 //Using nested IF instead of &&
295 if (addr.husnr < entry.tilHusnr || (addr.husnr == entry.tilHusnr && addr.husnrbogstav.compareTo(entry.tilLitra) <= 0 )) {
296 updateAddress(addr, entry);
297 }
298 }
299 }
300 }
301 }
302
303
304
305 private void updateAddress(Address addr, AddressSourceEntry entry) {
306 if ( addr.distributor != null && addr.distributor.equals(entry.distributor) == false) {
307 rejectedEntries.add(entry);
308 logger.info("Afviser " + entry);
309 return;
310 }
311
312
313
314 if (entry.ugedage.mandag) {
315 if (addr.visitedMan == false) {
316 addr.visitedMan = true;
317 if (addr.ruteMandag == null) {
318 addr.stateMan = AddressState.OPENED;
319 addr.ruteMandag = entry.rute;
320 addr.korelisteMandag = entry.koreliste;
321 } else {
322
323 if (StringUtils.equals(addr.ruteMandag, entry.rute) == false || StringUtils.equals(addr.korelisteMandag, entry.koreliste) == false) {
324 addr.stateMan = AddressState.MODIFIED;
325 addr.ruteMandag = entry.rute;
326 addr.korelisteMandag = entry.koreliste;
327 }
328 }
329 } else {
330 duplicateCount++;
331 logger.info( "Double visit monday " + addr);
332 return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters
333 }
334
335 }
336
337
338 if (entry.ugedage.tirsdag) {
339 if (addr.visitedTir == false) {
340 addr.visitedTir = true;
341 if (addr.ruteTirsdag == null) {
342 addr.stateTir = AddressState.OPENED;
343 addr.ruteTirsdag = entry.rute;
344 addr.korelisteTirsdag = entry.koreliste;
345 } else {
346
347 if (StringUtils.equals(addr.ruteTirsdag, entry.rute) == false || StringUtils.equals(addr.korelisteTirsdag, entry.koreliste) == false) {
348 addr.stateTir = AddressState.MODIFIED;
349 addr.ruteTirsdag = entry.rute;
350 addr.korelisteTirsdag = entry.koreliste;
351 }
352 }
353 } else {
354 duplicateCount++;
355 logger.info( "Double visit tuesday " + addr);
356 return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters
357 }
358 }
359
360 if (entry.ugedage.onsdag) {
361 if (addr.visitedOns == false) {
362 addr.visitedOns = true;
363 if (addr.ruteOnsdag == null) {
364 addr.stateOns = AddressState.OPENED;
365 addr.ruteOnsdag = entry.rute;
366 addr.korelisteOnsdag = entry.koreliste;
367 } else {
368
369 if (StringUtils.equals(addr.ruteOnsdag, entry.rute) == false || StringUtils.equals(addr.korelisteOnsdag, entry.koreliste) == false) {
370 addr.stateOns = AddressState.MODIFIED;
371 addr.ruteOnsdag = entry.rute;
372 addr.korelisteOnsdag = entry.koreliste;
373 }
374 }
375 } else {
376 duplicateCount++;
377 logger.info( "Double visit wednesday " + addr);
378 return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters
379 }
380 }
381
382
383 if (entry.ugedage.torsdag) {
384 if (addr.visitedTor == false) {
385 addr.visitedTor = true;
386 if (addr.ruteTorsdag == null) {
387 addr.stateTor = AddressState.OPENED;
388 addr.ruteTorsdag = entry.rute;
389 addr.korelisteTorsdag = entry.koreliste;
390 } else {
391
392
393 if (StringUtils.equals(addr.ruteTorsdag, entry.rute) == false || StringUtils.equals(addr.korelisteTorsdag, entry.koreliste) == false) {
394 addr.stateTor = AddressState.MODIFIED;
395 addr.ruteTorsdag = entry.rute;
396 addr.korelisteTorsdag = entry.koreliste;
397 }
398 }
399 } else {
400 duplicateCount++;
401 logger.info( "Double visit thursday " + addr);
402 return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters
403 }
404 }
405
406
407 if (entry.ugedage.fredag) {
408 if (addr.visitedFre == false) {
409 addr.visitedFre = true;
410 if (addr.ruteFredag == null) {
411 addr.stateFre = AddressState.OPENED;
412 addr.ruteFredag = entry.rute;
413 addr.korelisteFredag = entry.koreliste;
414 } else {
415
416 if (StringUtils.equals(addr.ruteFredag, entry.rute) == false || StringUtils.equals(addr.korelisteFredag, entry.koreliste) == false) {
417 addr.stateFre = AddressState.MODIFIED;
418 addr.ruteFredag = entry.rute;
419 addr.korelisteFredag = entry.koreliste;
420 }
421 }
422 } else {
423 duplicateCount++;
424 logger.info( "Double visit friday " + addr);
425 return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters
426 }
427 }
428
429
430 if (entry.ugedage.lordag) {
431 if (addr.visitedLor == false) {
432 addr.visitedLor = true;
433 if (addr.ruteLordag == null) {
434 addr.stateLor = AddressState.OPENED;
435 addr.ruteLordag = entry.rute;
436 addr.korelisteLordag = entry.koreliste;
437 } else {
438
439
440 if (StringUtils.equals(addr.ruteLordag, entry.rute) == false || StringUtils.equals(addr.korelisteLordag, entry.koreliste) == false) {
441 addr.stateLor = AddressState.MODIFIED;
442 addr.ruteLordag = entry.rute;
443 addr.korelisteLordag = entry.koreliste;
444 }
445 }
446 } else {
447 duplicateCount++;
448 logger.info( "Double visit saturday " + addr);
449 return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters
450 }
451 }
452
453
454 if (entry.ugedage.sondag) {
455 if (addr.visitedSon == false) {
456 addr.visitedSon = true;
457 if (addr.ruteSondag == null) {
458 addr.stateSon = AddressState.OPENED;
459 addr.ruteSondag = entry.rute;
460 addr.korelisteSondag = entry.koreliste;
461 } else {
462
463 if (StringUtils.equals(addr.ruteSondag, entry.rute) == false || StringUtils.equals(addr.korelisteSondag, entry.koreliste) == false) {
464 addr.stateSon = AddressState.MODIFIED;
465 addr.ruteSondag = entry.rute;
466 addr.korelisteSondag = entry.koreliste;
467 }
468 }
469 } else {
470 duplicateCount++;
471 logger.info( "Double visit sunday " + addr);
472 return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters
473 }
474 }
475
476
477
478 //addr.visited = true;
479 if (addr.state == AddressState.NOT_CHANGED) {
480 if (addr.distributor == null) {
481 addr.state = AddressState.OPENED;
482 addr.distributor = entry.distributor;
483 } else {
484 if (addr.stateMan != AddressState.NOT_CHANGED || addr.stateTir != AddressState.NOT_CHANGED
485 || addr.stateOns != AddressState.NOT_CHANGED || addr.stateTor != AddressState.NOT_CHANGED
486 || addr.stateFre != AddressState.NOT_CHANGED || addr.stateLor != AddressState.NOT_CHANGED || addr.stateSon != AddressState.NOT_CHANGED ){
487
488 addr.state = AddressState.MODIFIED;
489 }
490 }
491 }
492
493 updateDbkBane(addr);
494
495 }
496
497
498
499 private void createFromEntry(AddressSourceEntry entry) {
500 if (entry.husnr >= 1000) { //husnumre i DK må kun være på 3 cifre
501 return;
502 }
503
504 if (entry.husnr == 999 || entry.husnr == 998) { //frasorter de to højest gyldige husnr - det er typisk special adresser
505 return;
506 }
507
508 if (entry.litra.length() > 1) { //litra må kun være på 1 tegn
509 return;
510 }
511 if (entry.litra.length() == 1) {
512 char litra = entry.litra.charAt(0);
513 if ( Character.isLetter(litra) == false ) {
514 return;
515 }
516
517 }
518
519 if (entry.kommunekode == 0 || entry.vejkode == 0) {
520 return;
521 }
522 if (entry.kommunekode < 100) {
523 return;
524 }
525
526 if (entry.vejkode > 9999) {
527 return;
528 }
529
530 if (entry.gadeid == 0) {
531 return;
532 }
533
534 String gadeidStr = Integer.toString(entry.gadeid);
535 if (gadeidStr.length() != 9) {
536 return;
537 }
538 if (Short.parseShort(gadeidStr.substring(0, 4)) != entry.postnr) {
539 return; //gadeid / postnr mismatch
540 }
541
542
543
544 logger.info("Opretter adresse ud fra " + entry);
545
546 Address addr = new Address();
547 addr.state = AddressState.CREATED;
548
549 addr.distributor = entry.distributor;
550 addr.gadeid = entry.gadeid;
551 addr.kommunekode = entry.kommunekode;
552 addr.vejkode = entry.vejkode;
553 addr.vejnavn = entry.vejnavn;
554 addr.husnr = entry.husnr;
555 addr.husnrbogstav = entry.litra;
556 addr.postnr = entry.postnr;
557
558
559 updateAddress(addr, entry);
560
561
562 // Nu er det nye adresse object oprettet - nu skal det gemmes i søge strukturen og totallisten
563 TreeMap<Short, ArrayList<Address>> gade = searchStructure.get( addr.gadeid );
564
565 if (gade == null) {
566 gade = new TreeMap<Short,ArrayList<Address>>();
567 searchStructure.put(addr.gadeid, gade);
568 }
569
570 ArrayList<Address> litraList = gade.get(addr.husnr);
571 if (litraList == null) {
572 litraList = new ArrayList<Address>();
573 gade.put(addr.husnr, litraList);
574 }
575
576 litraList.add(addr);
577 addressList.add(addr);
578
579 }
580
581
582
583 /*
584 DAO:
585 UPDATE fulddaekning.adressetabel
586 SET dbkbane = case
587 when substr(korelisteMa,1,2) IN ('07','10','11','12','14','15','16','18','19','20') then 205 #DAO-BRA
588 when substr(korelisteMa,1,2) BETWEEN 24 and 30 then 201 #DAO-Ovrige
589 else 202 #dao syd (52-99=
590 END
591 WHERE distributor='DAO'
592 AND ruteMa is not null;
593
594 FD: Altid 200
595 NS: Altid 204
596 BK: 195,196,197,198
597 */
598
599 private void updateDbkBane(Address a) {
600 List<Short> daoBane205 = Arrays.asList( new Short[] {7,10,11,12,12,14,15,16,18,19,20} );
601
602
603 Short bane = null;
604
605 switch (a.distributor) {
606 case "BK":
607 bane = dbkBaneMap.get(a.postnr);
608 break;
609 case "DAO":
610 String koreliste = MiscUtils.firstNonNull(a.korelisteMandag,a.korelisteLordag, a.korelisteSondag); //DAO har kun 3 dækningstyper
611 short first2 = Short.parseShort( koreliste.substring(0,2) );
612 if ( daoBane205.contains(first2) ) {
613 bane = 205;
614 } else if (first2 >= 24 && first2<=30) {
615 bane = 201;
616 } else {
617 bane = 202;
618 }
619
620 break;
621 case "FD":
622 bane = 200;
623 break;
624 case "NS":
625 bane = 204;
626 break;
627 default:
628 throw new RuntimeException("Ukendt distributor: " + a.distributor); //Silence findBugs
629 }
630
631 if (bane == null) {
632 throw new RuntimeException("Ukendt bane for postnr: " + a.postnr);
633 }
634
635 if (a.dbkBane != bane) {
636 a.dbkBane = bane;
637 if (a.state ==AddressState.NOT_CHANGED) {
638 a.state = AddressState.MODIFIED;
639 }
640 }
641 }
642
643 public List<Address> getAddressList() {
644 return Collections.unmodifiableList(addressList);
645 }
646
647 public Map<Integer,String> getUnknownStreets() {
648 return Collections.unmodifiableMap( unknownStreets );
649 }
650
651 public int getDuplicateCount() {
652 return duplicateCount;
653 }
654
655 public int getRejectedCount() {
656 return rejectedEntries.size();
657 }
658
659 }

  ViewVC Help
Powered by ViewVC 1.1.20