/[projects]/dao/DaoAdresseVedligehold/src/main/java/dk/daoas/adressevedligehold/coveragefileupload/AddressManager.java
ViewVC logotype

Contents of /dao/DaoAdresseVedligehold/src/main/java/dk/daoas/adressevedligehold/coveragefileupload/AddressManager.java

Parent Directory Parent Directory | Revision Log Revision Log


Revision 3113 - (show annotations) (download)
Tue Sep 20 08:17:12 2016 UTC (7 years, 7 months ago) by torben
File size: 19776 byte(s)
Only trigger reload if there were any changes
1 package dk.daoas.adressevedligehold.coveragefileupload;
2
3 import java.sql.SQLException;
4 import java.util.ArrayList;
5 import java.util.Arrays;
6 import java.util.Collections;
7 import java.util.List;
8 import java.util.Map;
9 import java.util.Map.Entry;
10 import java.util.TreeMap;
11
12 import org.apache.commons.lang3.StringUtils;
13
14 import dk.daoas.adressevedligehold.beans.Address;
15 import dk.daoas.adressevedligehold.beans.Address.AddressState;
16 import dk.daoas.adressevedligehold.coveragefileupload.AddressSourceEntry.EntryType;
17 import dk.daoas.adressevedligehold.db.DatabaseCoverageUpdate;
18 import dk.daoas.adressevedligehold.tasks.TaskLogger;
19 import dk.daoas.adressevedligehold.util.DeduplicateHelper;
20 import dk.daoas.adressevedligehold.util.MiscUtils;
21 import dk.daoas.adressevedligehold.util.TimingHelper;
22
23 /*
24 *
25 */
26
27 public class AddressManager {
28
29 private TaskLogger logger = TaskLogger.getInstance();
30
31 private int duplicateCount;
32
33 List<Address> addressList;
34
35 Map<Integer, TreeMap<Short, ArrayList<Address>> > searchStructure;
36
37 ArrayList<AddressSourceEntry> rejectedEntries = new ArrayList<AddressSourceEntry>();
38
39 Map<Integer,String> unknownStreets = new TreeMap<Integer, String>();
40
41 Map<Short,String> dbkBaneMap = new TreeMap<Short,String>();
42
43 public AddressManager() throws SQLException {
44 DatabaseCoverageUpdate db = new DatabaseCoverageUpdate();
45
46 searchStructure = new TreeMap<Integer, TreeMap<Short, ArrayList<Address>> >();
47
48 addressList = db.getAllAdresses();
49
50 DeduplicateHelper<Integer> intHelper = new DeduplicateHelper<Integer>();
51 DeduplicateHelper<Short> shortHelper = new DeduplicateHelper<Short>();
52
53 List<ArrayList<Address>> arraylistCache = new ArrayList<ArrayList<Address>>();
54
55 TimingHelper timer = new TimingHelper();
56
57 for (Address a : addressList) {
58 Integer gadeid = intHelper.getInstance( a.gadeid );
59 Short husnr = shortHelper.getInstance( a.husnr );
60 TreeMap<Short, ArrayList<Address>> gade = searchStructure.get( gadeid );
61
62 if (a.dbkBane != null && a.dbkBane.length() > 0 && a.postnr<=4999) {
63 String bane = dbkBaneMap.get(a.postnr);
64 if (bane == null) {
65 dbkBaneMap.put(a.postnr, a.dbkBane);
66 }
67 }
68
69 if (gade == null) {
70 gade = new TreeMap<Short,ArrayList<Address>>();
71 searchStructure.put(gadeid, gade);
72 }
73
74 ArrayList<Address> litraList = gade.get(husnr);
75 if (litraList == null) {
76 litraList = new ArrayList<Address>();
77 gade.put(husnr, litraList);
78 arraylistCache.add(litraList);
79 }
80
81 litraList.add(a);
82 }
83
84 for (ArrayList<Address> list : arraylistCache) {
85 list.trimToSize();
86 }
87
88 logger.info("AddressManager ready, elapsed " + timer.getElapsed() + "ms");
89
90
91 }
92
93 @SuppressWarnings("PMD.CollapsibleIfStatements")
94 public int closeUnvisitedAddresses(String distributor, EntryUgedage ugedage) {
95
96 int totalClosed = 0;
97
98 for (Address addr : addressList) {
99 if (addr.distributor == null)
100 continue;
101
102 if (addr.distributor.equals(distributor) == false)//irrelevant for denne indlæsning
103 continue;
104
105 int closedCount = 0;
106
107 if (addr.visitedMan == false && ugedage.mandag) {
108 if (addr.ruteMandag != null) {
109 addr.ruteMandag = null;
110 addr.korelisteMandag = null;
111 addr.stateMan = AddressState.CLOSED;
112 closedCount++;
113 }
114 }
115 if (addr.visitedTir == false && ugedage.tirsdag) {
116 if (addr.ruteTirsdag != null) {
117 addr.ruteTirsdag = null;
118 addr.korelisteTirsdag = null;
119 addr.stateTir = AddressState.CLOSED;
120 closedCount++;
121 }
122 }
123
124 if (addr.visitedOns == false && ugedage.onsdag) {
125 if (addr.ruteOnsdag != null) {
126 addr.ruteOnsdag = null;
127 addr.korelisteOnsdag = null;
128 addr.stateOns = AddressState.CLOSED;
129 closedCount++;
130 }
131 }
132
133 if (addr.visitedTor == false && ugedage.torsdag) {
134 if (addr.ruteTorsdag != null) {
135 addr.ruteTorsdag = null;
136 addr.korelisteTorsdag = null;
137 addr.stateTor = AddressState.CLOSED;
138 closedCount++;
139 }
140 }
141
142 if (addr.visitedFre == false && ugedage.fredag) {
143 if (addr.ruteFredag != null) {
144 addr.ruteFredag = null;
145 addr.korelisteFredag = null;
146 addr.stateFre = AddressState.CLOSED;
147 closedCount++;
148 }
149 }
150
151 if (addr.visitedLor == false && ugedage.lordag) {
152 if (addr.ruteLordag != null) {
153 addr.ruteLordag = null;
154 addr.korelisteLordag = null;
155 addr.stateLor = AddressState.CLOSED;
156 closedCount++;
157 }
158 }
159
160 if (addr.visitedSon == false && ugedage.sondag) {
161 if (addr.ruteSondag != null) {
162 addr.ruteSondag = null;
163 addr.korelisteSondag = null;
164 addr.stateSon = AddressState.CLOSED;
165 closedCount++;
166 }
167 }
168
169 if (addr.state == AddressState.NOT_CHANGED && closedCount>0) {
170 addr.state = AddressState.CLOSED;
171 }
172
173 if (closedCount>0) {
174 totalClosed++;
175 }
176
177 }
178
179 return totalClosed;
180 }
181
182
183 public void dumpChanged() {
184 int count = 0;
185 for (Address addr : addressList) {
186 if (addr.state == AddressState.NOT_CHANGED) {
187 continue;
188 }
189 /*if (addr.stateMan == AddressState.NOT_CHANGED || addr.stateMan == AddressState.CREATED) {
190 continue;
191 }*/
192
193 if (addr.stateMan != AddressState.MODIFIED ) {
194 continue;
195 }
196
197 logger.info(addr.state + ": " + addr.toStringExtended() );
198 logger.info(addr.getRuteString() );
199
200 if (count++ > 50)
201 return;
202 }
203 }
204
205 public int writeBackChanges() throws Exception{
206 logger.info("Writing back changes");
207 TimingHelper timer = new TimingHelper();
208
209 List<Address> updates = new ArrayList<Address>(1024*1024);
210 List<Address> inserts = new ArrayList<Address>(16*1024);
211
212 for (Address addr : addressList) {
213 if (addr.state == AddressState.CREATED) {
214 inserts.add(addr);
215 }
216 if (addr.state == AddressState.MODIFIED || addr.state == AddressState.OPENED || addr.state == AddressState.CLOSED) {
217 updates.add(addr);
218 }
219 }
220
221 DatabaseCoverageUpdate db = new DatabaseCoverageUpdate();
222
223 db.updateAddresses(updates);
224 db.saveNewAddresses(inserts);
225
226
227
228 logger.info("Writeback done: elapsed " + timer.getElapsed() + "ms.");
229
230 return updates.size() + inserts.size();
231 }
232
233
234 public void visit(AddressSourceEntry entry) {
235 if (entry.type == EntryType.TypeSingleAddress) {
236 visitSingle(entry);
237 } else {
238 visitRange(entry);
239 }
240 }
241
242 private void visitSingle(AddressSourceEntry entry) {
243 TreeMap<Short, ArrayList<Address>> gade = searchStructure.get( entry.gadeid );
244 if (gade == null) {
245 /* No need to register unknown streets, since we create all entries that we can
246 if (! unknownStreets.containsKey(entry.gadeid)) {
247 unknownStreets.put(entry.gadeid, entry.vejnavn);
248 }*/
249
250 createFromEntry(entry); // if we get here there was no match - so we need to create it
251 return;
252 }
253
254 ArrayList<Address> litraList = gade.get(entry.husnr);
255 if (litraList == null) {
256 createFromEntry(entry); // if we get here there was no match - so we need to create it
257 return;
258 }
259 boolean found = false;
260 for (Address addr : litraList) {
261 if (addr.husnrbogstav.equals(entry.litra) ) {
262 updateAddress(addr, entry);
263 found = true; // 1 visit should be enough but as long as there's duplicates on gadeid+husnr+litra we will visit them all
264 //when the issue with duplicates is resolved this should be reverted to a return or break
265 }
266 }
267 if (found == false) {
268 createFromEntry(entry); // if we get here there was no match - so we need to create it
269 }
270 }
271
272 @SuppressWarnings("PMD.CollapsibleIfStatements")
273 private void visitRange(AddressSourceEntry entry) {
274
275 TreeMap<Short, ArrayList<Address>> gade = searchStructure.get( entry.gadeid );
276 if (gade == null) {
277 if (! unknownStreets.containsKey(entry.gadeid)) {
278 unknownStreets.put(entry.gadeid, entry.vejnavn);
279 }
280
281 logger.info("[Range] Ukendt gadeID " + entry);
282 return;
283 }
284
285 for (Entry<Short, ArrayList<Address>> husnrEntry: gade.entrySet()) {
286
287 short husnummer = husnrEntry.getKey();
288
289 if ( (entry.husnr % 2) != (husnummer%2) ){ //lige/ulige skal passe sammen
290 continue;
291 }
292
293 if ( husnummer < entry.husnr || husnummer > entry.tilHusnr) {
294 continue;
295 }
296
297 ArrayList<Address> litraList = husnrEntry.getValue();
298
299 /* a=adressetabel u=input
300 * and (a.husnr>u.FraHusNr OR (a.husnr=u.FraHusNr AND a.HusnrBogstav >= u.FraBog))
301 * AND (a.husnr<u.TilHusNr OR (a.husnr=u.TilHusNr AND a.HusnrBogstav <= u.TilBog))
302 */
303
304
305 for (Address addr: litraList) {
306 if (addr.husnr > entry.husnr || (addr.husnr == entry.husnr && addr.husnrbogstav.compareTo(entry.litra) >= 0 )) {
307 //Using nested IF instead of &&
308 if (addr.husnr < entry.tilHusnr || (addr.husnr == entry.tilHusnr && addr.husnrbogstav.compareTo(entry.tilLitra) <= 0 )) {
309 updateAddress(addr, entry);
310 }
311 }
312 }
313 }
314 }
315
316
317
318 private void updateAddress(Address addr, AddressSourceEntry entry) {
319 if ( addr.distributor != null && addr.distributor.equals(entry.distributor) == false) {
320 rejectedEntries.add(entry);
321 logger.info("Afviser " + entry);
322 return;
323 }
324
325
326
327 if (entry.ugedage.mandag) {
328 if (addr.visitedMan == false) {
329 addr.visitedMan = true;
330 if (addr.ruteMandag == null) {
331 addr.stateMan = AddressState.OPENED;
332 addr.ruteMandag = entry.rute;
333 addr.korelisteMandag = entry.koreliste;
334 } else {
335
336 if (StringUtils.equals(addr.ruteMandag, entry.rute) == false || StringUtils.equals(addr.korelisteMandag, entry.koreliste) == false) {
337 addr.stateMan = AddressState.MODIFIED;
338 addr.ruteMandag = entry.rute;
339 addr.korelisteMandag = entry.koreliste;
340 }
341 }
342 } else {
343 duplicateCount++;
344 logger.info( "Double visit monday " + addr);
345 return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters
346 }
347
348 }
349
350
351 if (entry.ugedage.tirsdag) {
352 if (addr.visitedTir == false) {
353 addr.visitedTir = true;
354 if (addr.ruteTirsdag == null) {
355 addr.stateTir = AddressState.OPENED;
356 addr.ruteTirsdag = entry.rute;
357 addr.korelisteTirsdag = entry.koreliste;
358 } else {
359
360 if (StringUtils.equals(addr.ruteTirsdag, entry.rute) == false || StringUtils.equals(addr.korelisteTirsdag, entry.koreliste) == false) {
361 addr.stateTir = AddressState.MODIFIED;
362 addr.ruteTirsdag = entry.rute;
363 addr.korelisteTirsdag = entry.koreliste;
364 }
365 }
366 } else {
367 duplicateCount++;
368 logger.info( "Double visit tuesday " + addr);
369 return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters
370 }
371 }
372
373 if (entry.ugedage.onsdag) {
374 if (addr.visitedOns == false) {
375 addr.visitedOns = true;
376 if (addr.ruteOnsdag == null) {
377 addr.stateOns = AddressState.OPENED;
378 addr.ruteOnsdag = entry.rute;
379 addr.korelisteOnsdag = entry.koreliste;
380 } else {
381
382 if (StringUtils.equals(addr.ruteOnsdag, entry.rute) == false || StringUtils.equals(addr.korelisteOnsdag, entry.koreliste) == false) {
383 addr.stateOns = AddressState.MODIFIED;
384 addr.ruteOnsdag = entry.rute;
385 addr.korelisteOnsdag = entry.koreliste;
386 }
387 }
388 } else {
389 duplicateCount++;
390 logger.info( "Double visit wednesday " + addr);
391 return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters
392 }
393 }
394
395
396 if (entry.ugedage.torsdag) {
397 if (addr.visitedTor == false) {
398 addr.visitedTor = true;
399 if (addr.ruteTorsdag == null) {
400 addr.stateTor = AddressState.OPENED;
401 addr.ruteTorsdag = entry.rute;
402 addr.korelisteTorsdag = entry.koreliste;
403 } else {
404
405
406 if (StringUtils.equals(addr.ruteTorsdag, entry.rute) == false || StringUtils.equals(addr.korelisteTorsdag, entry.koreliste) == false) {
407 addr.stateTor = AddressState.MODIFIED;
408 addr.ruteTorsdag = entry.rute;
409 addr.korelisteTorsdag = entry.koreliste;
410 }
411 }
412 } else {
413 duplicateCount++;
414 logger.info( "Double visit thursday " + addr);
415 return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters
416 }
417 }
418
419
420 if (entry.ugedage.fredag) {
421 if (addr.visitedFre == false) {
422 addr.visitedFre = true;
423 if (addr.ruteFredag == null) {
424 addr.stateFre = AddressState.OPENED;
425 addr.ruteFredag = entry.rute;
426 addr.korelisteFredag = entry.koreliste;
427 } else {
428
429 if (StringUtils.equals(addr.ruteFredag, entry.rute) == false || StringUtils.equals(addr.korelisteFredag, entry.koreliste) == false) {
430 addr.stateFre = AddressState.MODIFIED;
431 addr.ruteFredag = entry.rute;
432 addr.korelisteFredag = entry.koreliste;
433 }
434 }
435 } else {
436 duplicateCount++;
437 logger.info( "Double visit friday " + addr);
438 return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters
439 }
440 }
441
442
443 if (entry.ugedage.lordag) {
444 if (addr.visitedLor == false) {
445 addr.visitedLor = true;
446 if (addr.ruteLordag == null) {
447 addr.stateLor = AddressState.OPENED;
448 addr.ruteLordag = entry.rute;
449 addr.korelisteLordag = entry.koreliste;
450 } else {
451
452
453 if (StringUtils.equals(addr.ruteLordag, entry.rute) == false || StringUtils.equals(addr.korelisteLordag, entry.koreliste) == false) {
454 addr.stateLor = AddressState.MODIFIED;
455 addr.ruteLordag = entry.rute;
456 addr.korelisteLordag = entry.koreliste;
457 }
458 }
459 } else {
460 duplicateCount++;
461 logger.info( "Double visit saturday " + addr);
462 return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters
463 }
464 }
465
466
467 if (entry.ugedage.sondag) {
468 if (addr.visitedSon == false) {
469 addr.visitedSon = true;
470 if (addr.ruteSondag == null) {
471 addr.stateSon = AddressState.OPENED;
472 addr.ruteSondag = entry.rute;
473 addr.korelisteSondag = entry.koreliste;
474 } else {
475
476 if (StringUtils.equals(addr.ruteSondag, entry.rute) == false || StringUtils.equals(addr.korelisteSondag, entry.koreliste) == false) {
477 addr.stateSon = AddressState.MODIFIED;
478 addr.ruteSondag = entry.rute;
479 addr.korelisteSondag = entry.koreliste;
480 }
481 }
482 } else {
483 duplicateCount++;
484 logger.info( "Double visit sunday " + addr);
485 return; // if the entry is duplicate on one day that it covers - then it will be as well on all the othters
486 }
487 }
488
489
490
491 //addr.visited = true;
492 if (addr.state == AddressState.NOT_CHANGED) {
493 if (addr.distributor == null) {
494 addr.state = AddressState.OPENED;
495 addr.distributor = entry.distributor;
496 } else {
497 if (addr.stateMan != AddressState.NOT_CHANGED || addr.stateTir != AddressState.NOT_CHANGED
498 || addr.stateOns != AddressState.NOT_CHANGED || addr.stateTor != AddressState.NOT_CHANGED
499 || addr.stateFre != AddressState.NOT_CHANGED || addr.stateLor != AddressState.NOT_CHANGED || addr.stateSon != AddressState.NOT_CHANGED ){
500
501 addr.state = AddressState.MODIFIED;
502 }
503 }
504 }
505
506 updateDbkBane(addr);
507
508 }
509
510
511
512 private void createFromEntry(AddressSourceEntry entry) {
513 if (entry.husnr >= 1000) { //husnumre i DK må kun være på 3 cifre
514 return;
515 }
516
517 if (entry.husnr == 999 || entry.husnr == 998) { //frasorter de to højest gyldige husnr - det er typisk special adresser
518 return;
519 }
520
521 if (entry.litra.length() > 1) { //litra må kun være på 1 tegn
522 return;
523 }
524 if (entry.litra.length() == 1) {
525 char litra = entry.litra.charAt(0);
526 if ( Character.isLetter(litra) == false ) {
527 return;
528 }
529
530 }
531
532 if (entry.kommunekode == 0 || entry.vejkode == 0) {
533 return;
534 }
535 if (entry.kommunekode < 100) {
536 return;
537 }
538
539 if (entry.vejkode > 9999) {
540 return;
541 }
542
543 if (entry.gadeid == 0) {
544 return;
545 }
546
547 String gadeidStr = Integer.toString(entry.gadeid);
548 if (gadeidStr.length() != 9) {
549 return;
550 }
551 if (Short.parseShort(gadeidStr.substring(0, 4)) != entry.postnr) {
552 return; //gadeid / postnr mismatch
553 }
554
555
556
557 logger.info("Opretter adresse ud fra " + entry);
558
559 Address addr = new Address();
560 addr.state = AddressState.CREATED;
561
562 addr.distributor = entry.distributor;
563 addr.gadeid = entry.gadeid;
564 addr.kommunekode = entry.kommunekode;
565 addr.vejkode = entry.vejkode;
566 addr.vejnavn = entry.vejnavn;
567 addr.husnr = entry.husnr;
568 addr.husnrbogstav = entry.litra;
569 addr.postnr = entry.postnr;
570
571
572 updateAddress(addr, entry);
573
574
575 // Nu er det nye adresse object oprettet - nu skal det gemmes i søge strukturen og totallisten
576 TreeMap<Short, ArrayList<Address>> gade = searchStructure.get( addr.gadeid );
577
578 if (gade == null) {
579 gade = new TreeMap<Short,ArrayList<Address>>();
580 searchStructure.put(addr.gadeid, gade);
581 }
582
583 ArrayList<Address> litraList = gade.get(addr.husnr);
584 if (litraList == null) {
585 litraList = new ArrayList<Address>();
586 gade.put(addr.husnr, litraList);
587 }
588
589 litraList.add(addr);
590 addressList.add(addr);
591
592 }
593
594
595
596 /*
597 DAO:
598 UPDATE fulddaekning.adressetabel
599 SET dbkbane = case
600 when substr(korelisteMa,1,2) IN ('07','10','11','12','14','15','16','18','19','20') then 205 #DAO-BRA
601 when substr(korelisteMa,1,2) BETWEEN 24 and 30 then 201 #DAO-Ovrige
602 else 202 #dao syd (52-99=
603 END
604 WHERE distributor='DAO'
605 AND ruteMa is not null;
606
607 FD: Altid 200
608 NS: Altid 204
609 BK: 195,196,197,198
610 */
611
612 private void updateDbkBane(Address a) {
613 List<Short> daoBane205 = Arrays.asList( new Short[] {7,10,11,12,12,14,15,16,18,19,20} );
614
615
616 String bane = null;
617
618 switch (a.distributor) {
619 case "BK":
620 bane = dbkBaneMap.get(a.postnr);
621 break;
622 case "DAO":
623 String koreliste = MiscUtils.firstNonNull(a.korelisteMandag,a.korelisteLordag, a.korelisteSondag); //DAO har kun 3 dækningstyper
624 try {
625 short first2 = Short.parseShort( koreliste.substring(0,2) );
626 if ( daoBane205.contains(first2) ) { //Brabrand == 205 == W2
627 bane = "W2";
628 } else if (first2 >= 24 && first2<=30) { // DAO-Øvrig = 201 = W4
629 bane = "W4";
630 } else { // DAO-Syd= 202
631 bane = "W3";
632 }
633 } catch (NumberFormatException e) {
634 throw new RuntimeException("Error parsing koreliste '" + koreliste + "' for " + a);
635 }
636
637 break;
638 case "FD": // FD == Bane 200 = W5
639 bane = "W5";
640 break;
641 case "NS": // NS == Bane 204 == W1
642 bane = "204";
643 break;
644 default:
645 throw new RuntimeException("Ukendt distributor: " + a.distributor); //Silence findBugs
646 }
647
648 if (bane == null) {
649 throw new RuntimeException("Ukendt bane for postnr: " + a.postnr);
650 }
651
652 if (a.dbkBane != bane) {
653 a.dbkBane = bane;
654 if (a.state ==AddressState.NOT_CHANGED) {
655 a.state = AddressState.MODIFIED;
656 }
657 }
658 }
659
660 public List<Address> getAddressList() {
661 return Collections.unmodifiableList(addressList);
662 }
663
664 public Map<Integer,String> getUnknownStreets() {
665 return Collections.unmodifiableMap( unknownStreets );
666 }
667
668 public int getDuplicateCount() {
669 return duplicateCount;
670 }
671
672 public int getRejectedCount() {
673 return rejectedEntries.size();
674 }
675
676 }

  ViewVC Help
Powered by ViewVC 1.1.20