/[projects]/dao/DaoAdresseVedligehold/src/main/java/dk/daoas/adressevedligehold/coveragefileupload/AddressManager.java
ViewVC logotype

Contents of /dao/DaoAdresseVedligehold/src/main/java/dk/daoas/adressevedligehold/coveragefileupload/AddressManager.java

Parent Directory Parent Directory | Revision Log Revision Log


Revision 3084 - (show annotations) (download)
Fri Jul 29 09:16:03 2016 UTC (7 years, 9 months ago) by torben
File size: 19574 byte(s)
Check for number of closed address and alert if larger than 10.000
1 package dk.daoas.adressevedligehold.coveragefileupload;
2
3 import java.sql.SQLException;
4 import java.util.ArrayList;
5 import java.util.Arrays;
6 import java.util.Collections;
7 import java.util.List;
8 import java.util.Map;
9 import java.util.Map.Entry;
10 import java.util.TreeMap;
11
12 import org.apache.commons.lang3.StringUtils;
13
14 import dk.daoas.adressevedligehold.beans.Address;
15 import dk.daoas.adressevedligehold.beans.Address.AddressState;
16 import dk.daoas.adressevedligehold.coveragefileupload.AddressSourceEntry.EntryType;
17 import dk.daoas.adressevedligehold.db.DatabaseCoverageUpdate;
18 import dk.daoas.adressevedligehold.tasks.TaskLogger;
19 import dk.daoas.adressevedligehold.util.DeduplicateHelper;
20 import dk.daoas.adressevedligehold.util.MiscUtils;
21 import dk.daoas.adressevedligehold.util.TimingHelper;
22
23 /*
24 *
25 */
26
27 public class AddressManager {
28
29 private TaskLogger logger = TaskLogger.getInstance();
30
31 private int duplicateCount;
32
33 List<Address> addressList;
34
35 Map<Integer, TreeMap<Short, ArrayList<Address>> > searchStructure;
36
37 ArrayList<AddressSourceEntry> rejectedEntries = new ArrayList<AddressSourceEntry>();
38
39 Map<Integer,String> unknownStreets = new TreeMap<Integer, String>();
40
41 Map<Short,Short> dbkBaneMap = new TreeMap<Short,Short>();
42
43 public AddressManager() throws SQLException {
44 DatabaseCoverageUpdate db = new DatabaseCoverageUpdate();
45
46 searchStructure = new TreeMap<Integer, TreeMap<Short, ArrayList<Address>> >();
47
48 addressList = db.getAllAdresses();
49
50 DeduplicateHelper<Integer> intHelper = new DeduplicateHelper<Integer>();
51 DeduplicateHelper<Short> shortHelper = new DeduplicateHelper<Short>();
52
53 List<ArrayList<Address>> arraylistCache = new ArrayList<ArrayList<Address>>();
54
55 TimingHelper timer = new TimingHelper();
56
57 for (Address a : addressList) {
58 Integer gadeid = intHelper.getInstance( a.gadeid );
59 Short husnr = shortHelper.getInstance( a.husnr );
60 TreeMap<Short, ArrayList<Address>> gade = searchStructure.get( gadeid );
61
62 if (a.dbkBane > 0 && a.postnr<=4999) {
63 Short bane = dbkBaneMap.get(a.postnr);
64 if (bane == null) {
65 dbkBaneMap.put(a.postnr, a.dbkBane);
66 }
67 }
68
69 if (gade == null) {
70 gade = new TreeMap<Short,ArrayList<Address>>();
71 searchStructure.put(gadeid, gade);
72 }
73
74 ArrayList<Address> litraList = gade.get(husnr);
75 if (litraList == null) {
76 litraList = new ArrayList<Address>();
77 gade.put(husnr, litraList);
78 arraylistCache.add(litraList);
79 }
80
81 litraList.add(a);
82 }
83
84 for (ArrayList<Address> list : arraylistCache) {
85 list.trimToSize();
86 }
87
88 logger.info("AddressManager ready, elapsed " + timer.getElapsed() + "ms");
89
90
91 }
92
93 @SuppressWarnings("PMD.CollapsibleIfStatements")
94 public int closeUnvisitedAddresses(String distributor, EntryUgedage ugedage) {
95
96 int totalClosed = 0;
97
98 for (Address addr : addressList) {
99 if (addr.distributor == null)
100 continue;
101
102 if (addr.distributor.equals(distributor) == false)//irrelevant for denne indlæsning
103 continue;
104
105 int closedCount = 0;
106
107 if (addr.visitedMan == false && ugedage.mandag) {
108 if (addr.ruteMandag != null) {
109 addr.ruteMandag = null;
110 addr.korelisteMandag = null;
111 addr.stateMan = AddressState.CLOSED;
112 closedCount++;
113 }
114 }
115 if (addr.visitedTir == false && ugedage.tirsdag) {
116 if (addr.ruteTirsdag != null) {
117 addr.ruteTirsdag = null;
118 addr.korelisteTirsdag = null;
119 addr.stateTir = AddressState.CLOSED;
120 closedCount++;
121 }
122 }
123
124 if (addr.visitedOns == false && ugedage.onsdag) {
125 if (addr.ruteOnsdag != null) {
126 addr.ruteOnsdag = null;
127 addr.korelisteOnsdag = null;
128 addr.stateOns = AddressState.CLOSED;
129 closedCount++;
130 }
131 }
132
133 if (addr.visitedTor == false && ugedage.torsdag) {
134 if (addr.ruteTorsdag != null) {
135 addr.ruteTorsdag = null;
136 addr.korelisteTorsdag = null;
137 addr.stateTor = AddressState.CLOSED;
138 closedCount++;
139 }
140 }
141
142 if (addr.visitedFre == false && ugedage.fredag) {
143 if (addr.ruteFredag != null) {
144 addr.ruteFredag = null;
145 addr.korelisteFredag = null;
146 addr.stateFre = AddressState.CLOSED;
147 closedCount++;
148 }
149 }
150
151 if (addr.visitedLor == false && ugedage.lordag) {
152 if (addr.ruteLordag != null) {
153 addr.ruteLordag = null;
154 addr.korelisteLordag = null;
155 addr.stateLor = AddressState.CLOSED;
156 closedCount++;
157 }
158 }
159
160 if (addr.visitedSon == false && ugedage.sondag) {
161 if (addr.ruteSondag != null) {
162 addr.ruteSondag = null;
163 addr.korelisteSondag = null;
164 addr.stateSon = AddressState.CLOSED;
165 closedCount++;
166 }
167 }
168
169 if (addr.state == AddressState.NOT_CHANGED && closedCount>0) {
170 addr.state = AddressState.CLOSED;
171 }
172
173 if (closedCount>0) {
174 totalClosed++;
175 }
176
177 }
178
179 return totalClosed;
180 }
181
182
183 public void dumpChanged() {
184 int count = 0;
185 for (Address addr : addressList) {
186 if (addr.state == AddressState.NOT_CHANGED) {
187 continue;
188 }
189 /*if (addr.stateMan == AddressState.NOT_CHANGED || addr.stateMan == AddressState.CREATED) {
190 continue;
191 }*/
192
193 if (addr.stateMan != AddressState.MODIFIED ) {
194 continue;
195 }
196
197 logger.info(addr.state + ": " + addr.toStringExtended() );
198 logger.info(addr.getRuteString() );
199
200 if (count++ > 50)
201 return;
202 }
203 }
204
205 public void writeBackChanges() throws Exception{
206 logger.info("Writing back changes");
207 TimingHelper timer = new TimingHelper();
208
209 List<Address> updates = new ArrayList<Address>(1024*1024);
210 List<Address> inserts = new ArrayList<Address>(16*1024);
211
212 for (Address addr : addressList) {
213 if (addr.state == AddressState.CREATED) {
214 inserts.add(addr);
215 }
216 if (addr.state == AddressState.MODIFIED || addr.state == AddressState.OPENED || addr.state == AddressState.CLOSED) {
217 updates.add(addr);
218 }
219 }
220
221 DatabaseCoverageUpdate db = new DatabaseCoverageUpdate();
222
223 db.updateAddresses(updates);
224 db.saveNewAddresses(inserts);
225
226
227
228 logger.info("Writeback done: elapsed " + timer.getElapsed() + "ms.");
229 }
230
231
232 public void visit(AddressSourceEntry entry) {
233 if (entry.type == EntryType.TypeSingleAddress) {
234 visitSingle(entry);
235 } else {
236 visitRange(entry);
237 }
238 }
239
240 private void visitSingle(AddressSourceEntry entry) {
241 TreeMap<Short, ArrayList<Address>> gade = searchStructure.get( entry.gadeid );
242 if (gade == null) {
243 /* No need to register unknown streets, since we create all entries that we can
244 if (! unknownStreets.containsKey(entry.gadeid)) {
245 unknownStreets.put(entry.gadeid, entry.vejnavn);
246 }*/
247
248 createFromEntry(entry); // if we get here there was no match - so we need to create it
249 return;
250 }
251
252 ArrayList<Address> litraList = gade.get(entry.husnr);
253 if (litraList == null) {
254 createFromEntry(entry); // if we get here there was no match - so we need to create it
255 return;
256 }
257 boolean found = false;
258 for (Address addr : litraList) {
259 if (addr.husnrbogstav.equals(entry.litra) ) {
260 updateAddress(addr, entry);
261 found = true; // 1 visit should be enough but as long as there's duplicates on gadeid+husnr+litra we will visit them all
262 //when the issue with duplicates is resolved this should be reverted to a return or break
263 }
264 }
265 if (found == false) {
266 createFromEntry(entry); // if we get here there was no match - so we need to create it
267 }
268 }
269
270 @SuppressWarnings("PMD.CollapsibleIfStatements")
271 private void visitRange(AddressSourceEntry entry) {
272
273 TreeMap<Short, ArrayList<Address>> gade = searchStructure.get( entry.gadeid );
274 if (gade == null) {
275 if (! unknownStreets.containsKey(entry.gadeid)) {
276 unknownStreets.put(entry.gadeid, entry.vejnavn);
277 }
278
279 logger.info("[Range] Ukendt gadeID " + entry);
280 return;
281 }
282
283 for (Entry<Short, ArrayList<Address>> husnrEntry: gade.entrySet()) {
284
285 short husnummer = husnrEntry.getKey();
286
287 if ( (entry.husnr % 2) != (husnummer%2) ){ //lige/ulige skal passe sammen
288 continue;
289 }
290
291 if ( husnummer < entry.husnr || husnummer > entry.tilHusnr) {
292 continue;
293 }
294
295 ArrayList<Address> litraList = husnrEntry.getValue();
296
297 /* a=adressetabel u=input
298 * and (a.husnr>u.FraHusNr OR (a.husnr=u.FraHusNr AND a.HusnrBogstav >= u.FraBog))
299 * AND (a.husnr<u.TilHusNr OR (a.husnr=u.TilHusNr AND a.HusnrBogstav <= u.TilBog))
300 */
301
302
303 for (Address addr: litraList) {
304 if (addr.husnr > entry.husnr || (addr.husnr == entry.husnr && addr.husnrbogstav.compareTo(entry.litra) >= 0 )) {
305 //Using nested IF instead of &&
306 if (addr.husnr < entry.tilHusnr || (addr.husnr == entry.tilHusnr && addr.husnrbogstav.compareTo(entry.tilLitra) <= 0 )) {
307 updateAddress(addr, entry);
308 }
309 }
310 }
311 }
312 }
313
314
315
316 private void updateAddress(Address addr, AddressSourceEntry entry) {
317 if ( addr.distributor != null && addr.distributor.equals(entry.distributor) == false) {
318 rejectedEntries.add(entry);
319 logger.info("Afviser " + entry);
320 return;
321 }
322
323
324
325 if (entry.ugedage.mandag) {
326 if (addr.visitedMan == false) {
327 addr.visitedMan = true;
328 if (addr.ruteMandag == null) {
329 addr.stateMan = AddressState.OPENED;
330 addr.ruteMandag = entry.rute;
331 addr.korelisteMandag = entry.koreliste;
332 } else {
333
334 if (StringUtils.equals(addr.ruteMandag, entry.rute) == false || StringUtils.equals(addr.korelisteMandag, entry.koreliste) == false) {
335 addr.stateMan = AddressState.MODIFIED;
336 addr.ruteMandag = entry.rute;
337 addr.korelisteMandag = entry.koreliste;
338 }
339 }
340 } else {
341 duplicateCount++;
342 logger.info( "Double visit monday " + addr);
343 return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters
344 }
345
346 }
347
348
349 if (entry.ugedage.tirsdag) {
350 if (addr.visitedTir == false) {
351 addr.visitedTir = true;
352 if (addr.ruteTirsdag == null) {
353 addr.stateTir = AddressState.OPENED;
354 addr.ruteTirsdag = entry.rute;
355 addr.korelisteTirsdag = entry.koreliste;
356 } else {
357
358 if (StringUtils.equals(addr.ruteTirsdag, entry.rute) == false || StringUtils.equals(addr.korelisteTirsdag, entry.koreliste) == false) {
359 addr.stateTir = AddressState.MODIFIED;
360 addr.ruteTirsdag = entry.rute;
361 addr.korelisteTirsdag = entry.koreliste;
362 }
363 }
364 } else {
365 duplicateCount++;
366 logger.info( "Double visit tuesday " + addr);
367 return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters
368 }
369 }
370
371 if (entry.ugedage.onsdag) {
372 if (addr.visitedOns == false) {
373 addr.visitedOns = true;
374 if (addr.ruteOnsdag == null) {
375 addr.stateOns = AddressState.OPENED;
376 addr.ruteOnsdag = entry.rute;
377 addr.korelisteOnsdag = entry.koreliste;
378 } else {
379
380 if (StringUtils.equals(addr.ruteOnsdag, entry.rute) == false || StringUtils.equals(addr.korelisteOnsdag, entry.koreliste) == false) {
381 addr.stateOns = AddressState.MODIFIED;
382 addr.ruteOnsdag = entry.rute;
383 addr.korelisteOnsdag = entry.koreliste;
384 }
385 }
386 } else {
387 duplicateCount++;
388 logger.info( "Double visit wednesday " + addr);
389 return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters
390 }
391 }
392
393
394 if (entry.ugedage.torsdag) {
395 if (addr.visitedTor == false) {
396 addr.visitedTor = true;
397 if (addr.ruteTorsdag == null) {
398 addr.stateTor = AddressState.OPENED;
399 addr.ruteTorsdag = entry.rute;
400 addr.korelisteTorsdag = entry.koreliste;
401 } else {
402
403
404 if (StringUtils.equals(addr.ruteTorsdag, entry.rute) == false || StringUtils.equals(addr.korelisteTorsdag, entry.koreliste) == false) {
405 addr.stateTor = AddressState.MODIFIED;
406 addr.ruteTorsdag = entry.rute;
407 addr.korelisteTorsdag = entry.koreliste;
408 }
409 }
410 } else {
411 duplicateCount++;
412 logger.info( "Double visit thursday " + addr);
413 return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters
414 }
415 }
416
417
418 if (entry.ugedage.fredag) {
419 if (addr.visitedFre == false) {
420 addr.visitedFre = true;
421 if (addr.ruteFredag == null) {
422 addr.stateFre = AddressState.OPENED;
423 addr.ruteFredag = entry.rute;
424 addr.korelisteFredag = entry.koreliste;
425 } else {
426
427 if (StringUtils.equals(addr.ruteFredag, entry.rute) == false || StringUtils.equals(addr.korelisteFredag, entry.koreliste) == false) {
428 addr.stateFre = AddressState.MODIFIED;
429 addr.ruteFredag = entry.rute;
430 addr.korelisteFredag = entry.koreliste;
431 }
432 }
433 } else {
434 duplicateCount++;
435 logger.info( "Double visit friday " + addr);
436 return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters
437 }
438 }
439
440
441 if (entry.ugedage.lordag) {
442 if (addr.visitedLor == false) {
443 addr.visitedLor = true;
444 if (addr.ruteLordag == null) {
445 addr.stateLor = AddressState.OPENED;
446 addr.ruteLordag = entry.rute;
447 addr.korelisteLordag = entry.koreliste;
448 } else {
449
450
451 if (StringUtils.equals(addr.ruteLordag, entry.rute) == false || StringUtils.equals(addr.korelisteLordag, entry.koreliste) == false) {
452 addr.stateLor = AddressState.MODIFIED;
453 addr.ruteLordag = entry.rute;
454 addr.korelisteLordag = entry.koreliste;
455 }
456 }
457 } else {
458 duplicateCount++;
459 logger.info( "Double visit saturday " + addr);
460 return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters
461 }
462 }
463
464
465 if (entry.ugedage.sondag) {
466 if (addr.visitedSon == false) {
467 addr.visitedSon = true;
468 if (addr.ruteSondag == null) {
469 addr.stateSon = AddressState.OPENED;
470 addr.ruteSondag = entry.rute;
471 addr.korelisteSondag = entry.koreliste;
472 } else {
473
474 if (StringUtils.equals(addr.ruteSondag, entry.rute) == false || StringUtils.equals(addr.korelisteSondag, entry.koreliste) == false) {
475 addr.stateSon = AddressState.MODIFIED;
476 addr.ruteSondag = entry.rute;
477 addr.korelisteSondag = entry.koreliste;
478 }
479 }
480 } else {
481 duplicateCount++;
482 logger.info( "Double visit sunday " + addr);
483 return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters
484 }
485 }
486
487
488
489 //addr.visited = true;
490 if (addr.state == AddressState.NOT_CHANGED) {
491 if (addr.distributor == null) {
492 addr.state = AddressState.OPENED;
493 addr.distributor = entry.distributor;
494 } else {
495 if (addr.stateMan != AddressState.NOT_CHANGED || addr.stateTir != AddressState.NOT_CHANGED
496 || addr.stateOns != AddressState.NOT_CHANGED || addr.stateTor != AddressState.NOT_CHANGED
497 || addr.stateFre != AddressState.NOT_CHANGED || addr.stateLor != AddressState.NOT_CHANGED || addr.stateSon != AddressState.NOT_CHANGED ){
498
499 addr.state = AddressState.MODIFIED;
500 }
501 }
502 }
503
504 updateDbkBane(addr);
505
506 }
507
508
509
510 private void createFromEntry(AddressSourceEntry entry) {
511 if (entry.husnr >= 1000) { //husnumre i DK må kun være på 3 cifre
512 return;
513 }
514
515 if (entry.husnr == 999 || entry.husnr == 998) { //frasorter de to højest gyldige husnr - det er typisk special adresser
516 return;
517 }
518
519 if (entry.litra.length() > 1) { //litra må kun være på 1 tegn
520 return;
521 }
522 if (entry.litra.length() == 1) {
523 char litra = entry.litra.charAt(0);
524 if ( Character.isLetter(litra) == false ) {
525 return;
526 }
527
528 }
529
530 if (entry.kommunekode == 0 || entry.vejkode == 0) {
531 return;
532 }
533 if (entry.kommunekode < 100) {
534 return;
535 }
536
537 if (entry.vejkode > 9999) {
538 return;
539 }
540
541 if (entry.gadeid == 0) {
542 return;
543 }
544
545 String gadeidStr = Integer.toString(entry.gadeid);
546 if (gadeidStr.length() != 9) {
547 return;
548 }
549 if (Short.parseShort(gadeidStr.substring(0, 4)) != entry.postnr) {
550 return; //gadeid / postnr mismatch
551 }
552
553
554
555 logger.info("Opretter adresse ud fra " + entry);
556
557 Address addr = new Address();
558 addr.state = AddressState.CREATED;
559
560 addr.distributor = entry.distributor;
561 addr.gadeid = entry.gadeid;
562 addr.kommunekode = entry.kommunekode;
563 addr.vejkode = entry.vejkode;
564 addr.vejnavn = entry.vejnavn;
565 addr.husnr = entry.husnr;
566 addr.husnrbogstav = entry.litra;
567 addr.postnr = entry.postnr;
568
569
570 updateAddress(addr, entry);
571
572
573 // Nu er det nye adresse object oprettet - nu skal det gemmes i søge strukturen og totallisten
574 TreeMap<Short, ArrayList<Address>> gade = searchStructure.get( addr.gadeid );
575
576 if (gade == null) {
577 gade = new TreeMap<Short,ArrayList<Address>>();
578 searchStructure.put(addr.gadeid, gade);
579 }
580
581 ArrayList<Address> litraList = gade.get(addr.husnr);
582 if (litraList == null) {
583 litraList = new ArrayList<Address>();
584 gade.put(addr.husnr, litraList);
585 }
586
587 litraList.add(addr);
588 addressList.add(addr);
589
590 }
591
592
593
594 /*
595 DAO:
596 UPDATE fulddaekning.adressetabel
597 SET dbkbane = case
598 when substr(korelisteMa,1,2) IN ('07','10','11','12','14','15','16','18','19','20') then 205 #DAO-BRA
599 when substr(korelisteMa,1,2) BETWEEN 24 and 30 then 201 #DAO-Ovrige
600 else 202 #dao syd (52-99=
601 END
602 WHERE distributor='DAO'
603 AND ruteMa is not null;
604
605 FD: Altid 200
606 NS: Altid 204
607 BK: 195,196,197,198
608 */
609
610 private void updateDbkBane(Address a) {
611 List<Short> daoBane205 = Arrays.asList( new Short[] {7,10,11,12,12,14,15,16,18,19,20} );
612
613
614 Short bane = null;
615
616 switch (a.distributor) {
617 case "BK":
618 bane = dbkBaneMap.get(a.postnr);
619 break;
620 case "DAO":
621 String koreliste = MiscUtils.firstNonNull(a.korelisteMandag,a.korelisteLordag, a.korelisteSondag); //DAO har kun 3 dækningstyper
622 try {
623 short first2 = Short.parseShort( koreliste.substring(0,2) );
624 if ( daoBane205.contains(first2) ) {
625 bane = 205;
626 } else if (first2 >= 24 && first2<=30) {
627 bane = 201;
628 } else {
629 bane = 202;
630 }
631 } catch (NumberFormatException e) {
632 throw new RuntimeException("Error parsing koreliste '" + koreliste + "' for " + a);
633 }
634
635 break;
636 case "FD":
637 bane = 200;
638 break;
639 case "NS":
640 bane = 204;
641 break;
642 default:
643 throw new RuntimeException("Ukendt distributor: " + a.distributor); //Silence findBugs
644 }
645
646 if (bane == null) {
647 throw new RuntimeException("Ukendt bane for postnr: " + a.postnr);
648 }
649
650 if (a.dbkBane != bane) {
651 a.dbkBane = bane;
652 if (a.state ==AddressState.NOT_CHANGED) {
653 a.state = AddressState.MODIFIED;
654 }
655 }
656 }
657
658 public List<Address> getAddressList() {
659 return Collections.unmodifiableList(addressList);
660 }
661
662 public Map<Integer,String> getUnknownStreets() {
663 return Collections.unmodifiableMap( unknownStreets );
664 }
665
666 public int getDuplicateCount() {
667 return duplicateCount;
668 }
669
670 public int getRejectedCount() {
671 return rejectedEntries.size();
672 }
673
674 }

  ViewVC Help
Powered by ViewVC 1.1.20