2 |
|
|
3 |
import java.io.File; |
import java.io.File; |
4 |
import java.io.FileReader; |
import java.io.FileReader; |
5 |
|
import java.util.HashMap; |
6 |
import java.util.List; |
import java.util.List; |
7 |
|
import java.util.Map; |
8 |
|
import java.util.Queue; |
9 |
|
import java.util.Set; |
10 |
|
import java.util.concurrent.CyclicBarrier; |
11 |
|
import java.util.concurrent.Executors; |
12 |
|
import java.util.concurrent.ThreadFactory; |
13 |
|
import java.util.concurrent.ThreadPoolExecutor; |
14 |
|
import java.util.concurrent.atomic.AtomicInteger; |
15 |
import java.util.logging.FileHandler; |
import java.util.logging.FileHandler; |
16 |
import java.util.logging.Logger; |
import java.util.logging.Logger; |
17 |
import java.util.logging.SimpleFormatter; |
import java.util.logging.SimpleFormatter; |
18 |
import java.util.concurrent.*; |
|
19 |
|
import ags.utils.dataStructures.trees.thirdGenKD.KdTree; |
20 |
|
|
21 |
|
|
22 |
|
|
23 |
public class LookupMain { |
public class LookupMain { |
24 |
|
|
25 |
|
|
26 |
static final String CONFIG_FILENAME = "fulddaekning.properties"; |
static final String CONFIG_FILENAME = "fulddaekning.properties"; |
27 |
|
|
28 |
static boolean rename_tables; |
static boolean test_mode; |
29 |
static SafeProperties conf; |
static SafeProperties conf; |
30 |
|
|
31 |
static int max_workers; |
static int max_workers; |
32 |
static boolean verbose; |
static boolean verbose; |
33 |
|
|
34 |
static String distributor; |
static String distributor; |
35 |
|
|
36 |
final static Logger logger = Logger.getLogger( LookupMain.class.toString() ); |
final static Logger logger = Logger.getLogger( LookupMain.class.toString() ); |
37 |
|
|
|
|
|
|
static Statistik flestDaekkede = new Statistik(); |
|
|
static Statistik flestIkkeDaekkede = new Statistik(); |
|
|
static Statistik mestBrugteTid = new Statistik(); |
|
|
static Statistik stoersteDataset = new Statistik(); |
|
|
|
|
|
static Adresse[] alleDaekkedeAdresser; |
|
38 |
|
|
39 |
static ThreadPoolExecutor threadPool; |
static ThreadPoolExecutor threadPool; |
40 |
|
|
41 |
private static void setupThreadPool() { |
private static void setupThreadPool() { |
42 |
threadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(max_workers, new WorkerThreadFactory() ); |
threadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(max_workers, new WorkerThreadFactory() ); |
43 |
} |
} |
44 |
|
|
45 |
|
|
46 |
|
|
47 |
public static void main(String[] args) throws Exception { |
public static void main(String[] args) throws Exception { |
48 |
|
|
49 |
//Setup j.u.l Logger |
//Setup j.u.l Logger |
50 |
Logger root = Logger.getLogger(""); |
Logger root = Logger.getLogger(""); |
51 |
FileHandler fhandler = new FileHandler("fulddaekning.log"); // Ingen max størrelse, ingen rotation og ingen append |
FileHandler fhandler = new FileHandler("fulddaekning.log"); // Ingen max størrelse, ingen rotation og ingen append |
52 |
fhandler.setFormatter( new SimpleFormatter() ); |
fhandler.setFormatter( new SimpleFormatter() ); |
53 |
root.addHandler( fhandler ); |
root.addHandler( fhandler ); |
54 |
|
|
55 |
|
|
56 |
|
|
57 |
|
|
58 |
File confFile = new File( CONFIG_FILENAME ); |
File confFile = new File( CONFIG_FILENAME ); |
59 |
if (! confFile.exists() ) { |
if (! confFile.exists() ) { |
60 |
logger.warning("Config file not found: " + CONFIG_FILENAME); |
logger.warning("Config file not found: " + CONFIG_FILENAME); |
61 |
System.exit(1); |
System.exit(1); |
62 |
} |
} |
63 |
|
|
64 |
|
int antalIkkeDaekkede = -1; |
65 |
|
|
66 |
conf = new SafeProperties(); |
conf = new SafeProperties(); |
67 |
conf.load( new FileReader(confFile) ); |
conf.load( new FileReader(confFile) ); |
68 |
|
|
69 |
max_workers = Integer.parseInt( conf.getSafeProperty("MAX_WORKERS") ); |
|
70 |
if (max_workers <= 0) { |
|
|
logger.info("!!! AUTO-DETECT MAX_WORKERS !!!"); |
|
|
int cores = Runtime.getRuntime().availableProcessors(); |
|
|
cores -= 1;//Efterlad 1 core/cpu i reserve til systemet |
|
|
|
|
|
max_workers = Math.max(1, cores); //Dog skal der som minimum være 1 core til beregning |
|
|
|
|
|
} |
|
|
logger.info("Starting with MAX_WORKERS:" + max_workers); |
|
|
setupThreadPool(); |
|
|
|
|
|
|
|
71 |
verbose = Boolean.parseBoolean( conf.getSafeProperty("VERBOSE") ); |
verbose = Boolean.parseBoolean( conf.getSafeProperty("VERBOSE") ); |
72 |
logger.info("Starting with VERBOSE:" + verbose); |
logger.info("Starting with VERBOSE:" + verbose); |
73 |
|
|
74 |
rename_tables = Boolean.parseBoolean( conf.getSafeProperty("RENAMETABLES") ); |
test_mode = Boolean.parseBoolean( conf.getSafeProperty("TEST") ); |
75 |
logger.info("Starting with RENAMETABLES:" + rename_tables); |
logger.info("Starting with TEST:" + test_mode); |
76 |
|
|
77 |
distributor = conf.getSafeProperty("DISTRIBUTOR"); |
distributor = conf.getSafeProperty("DISTRIBUTOR"); |
78 |
distributor = distributor.toUpperCase(); |
distributor = distributor.toUpperCase(); |
79 |
logger.info("Starting for DISTRIBUTOR:" + distributor); |
logger.info("Starting for DISTRIBUTOR:" + distributor); |
80 |
|
|
81 |
Constants.init(distributor); |
Constants.init(distributor); |
82 |
Constants consts = Constants.getInstance(); |
Constants consts = Constants.getInstance(); |
83 |
|
|
84 |
|
max_workers = Integer.parseInt( conf.getSafeProperty("MAX_WORKERS") ); |
85 |
|
if (max_workers <= 0) { |
86 |
|
logger.info("!!! AUTO-DETECT MAX_WORKERS !!!"); |
87 |
|
int cores = Runtime.getRuntime().availableProcessors(); |
88 |
|
cores -= 1;//Efterlad 1 core/cpu i reserve til systemet |
89 |
|
|
90 |
|
max_workers = Math.max(1, cores); //Dog skal der som minimum være 1 core til beregning |
91 |
|
|
92 |
|
} |
93 |
|
if (test_mode) { |
94 |
|
max_workers = 1; |
95 |
|
} |
96 |
|
logger.info("Starting with MAX_WORKERS:" + max_workers); |
97 |
|
setupThreadPool(); |
98 |
|
|
99 |
|
|
100 |
Database db = new Database(conf); |
Database db = new Database(conf); |
101 |
|
Queue<Adresse> ikkeDaekkede = db.hentAlleIkkedaekkedeAdresser(consts.getMinPostnr(), consts.getMaxPostnr() ); |
102 |
|
antalIkkeDaekkede = ikkeDaekkede.size(); |
103 |
|
logger.info("Antal ikke-daekkede: " + antalIkkeDaekkede ); |
104 |
|
|
105 |
|
|
106 |
|
|
|
|
|
107 |
boolean testRun= false; |
boolean testRun= false; |
108 |
|
|
109 |
|
AtomicInteger antalFundne = new AtomicInteger(0); |
110 |
|
|
111 |
|
long beregnStart =0; |
112 |
long start = System.currentTimeMillis(); |
long start = System.currentTimeMillis(); |
113 |
|
int antalDaekkedeAdresser = -1; |
114 |
|
|
115 |
if (testRun == false) { |
if (testRun == false) { |
116 |
|
|
117 |
logger.info("Finder postnumre"); |
logger.info("Finder postnumre"); |
118 |
List<String> postnumre = db.hentPostnumre(); |
Set<Short> postnumre = db.hentPostnumreCache(); |
119 |
|
|
120 |
// Først validerer vi BBox på alle postnummre, for at undgå fuldt stop midt i beregningen |
// Først validerer vi BBox på alle postnummre, for at undgå fuldt stop midt i beregningen |
121 |
for(String postnr : postnumre) { // |
for(short postnr : postnumre) { // |
122 |
logger.info("Validerer BBox for " + postnr); |
logger.info("Validerer BBox for " + postnr); |
123 |
BoundingBox bbox = db.getBoundingbox(postnr); |
BoundingBox bbox = db.getBoundingbox(postnr); |
124 |
bbox.validateBbox(); |
bbox.validateBbox(); |
125 |
} |
} |
126 |
|
|
127 |
if (consts.doExtendedLookup()) { |
|
128 |
logger.info("Henter alle daekkede adresser"); |
logger.info("Henter alle daekkede adresser"); |
129 |
alleDaekkedeAdresser = db.hentAlleDaekkedeAdresser(); |
antalDaekkedeAdresser = db.hentAlleDaekkedeAdresser(); |
130 |
logger.info( "AlleDaekkedeAdresser.length=" + alleDaekkedeAdresser.length); |
logger.info( "AlleDaekkedeAdresser.length=" + antalDaekkedeAdresser ); |
131 |
|
|
132 |
|
Map<Short, List<Adresse>> addrHoList = db.getDaekkedeAdresserHO(); |
133 |
|
|
134 |
|
Map<Short, KdTree<Adresse>> hoTrees = new HashMap<Short,KdTree<Adresse>>(); |
135 |
|
|
136 |
|
for ( Map.Entry<Short, List<Adresse>> entry : addrHoList.entrySet() ) { |
137 |
|
short ho = entry.getKey(); |
138 |
|
List<Adresse> geopoints = entry.getValue(); |
139 |
|
|
140 |
|
logger.info("Opbygger KDTree for " + ho + " - antal=" + geopoints.size() ); |
141 |
|
|
142 |
|
int bucketSize = 96*12; |
143 |
|
|
144 |
|
KdTree<Adresse> addressTree = new KdTree<Adresse>( 3, bucketSize ); |
145 |
|
for(Adresse a: geopoints) { |
146 |
|
addressTree.addPoint(a.xyz, a); |
147 |
|
} |
148 |
|
|
149 |
|
hoTrees.put(ho, addressTree); |
150 |
} |
} |
151 |
|
|
|
|
|
|
//pre-check er ok - reset tmp tabel og start søgningen |
|
152 |
db.resetResultTable(); |
db.resetResultTable(); |
153 |
|
|
154 |
|
beregnStart = System.currentTimeMillis(); |
155 |
|
logger.info("Starter beregning"); |
156 |
|
|
157 |
|
//pre-check er ok - reset tmp tabel og start søgningen |
158 |
|
|
159 |
|
CyclicBarrier barrier = new CyclicBarrier(LookupMain.max_workers + 1); |
160 |
|
|
161 |
for(String postnr : postnumre) { |
for (int i=0; i<max_workers; i++) { |
162 |
Lookup lookup = new Lookup(postnr, db, threadPool); |
LookupWorker worker = new LookupWorker(i, barrier, ikkeDaekkede, hoTrees,antalFundne,db,verbose,consts); |
163 |
lookup.doLookup(); |
threadPool.submit( worker ); |
164 |
} |
} |
165 |
|
|
166 |
if (rename_tables) { |
barrier.await(); // Afvent at workerne bliver færdige |
167 |
|
|
168 |
|
logger.info("Calc is done - cleaning up remaining bits"); |
169 |
|
|
170 |
|
threadPool.shutdown(); //Calc is done now |
171 |
|
|
172 |
|
|
173 |
|
db.saveBatch(); |
174 |
|
|
175 |
|
if (test_mode == false) { |
176 |
db.renameResultTables(); |
db.renameResultTables(); |
177 |
} else { |
} else { |
178 |
logger.info( "Rename tables is disabled !!!" ); |
logger.info( "Rename tables is disabled !!!" ); |
179 |
} |
} |
180 |
|
|
181 |
|
|
182 |
|
|
183 |
} else { |
} else { |
184 |
/// Test |
/// Test |
185 |
|
/* |
186 |
db.resetResultTable(); |
db.resetResultTable(); |
187 |
|
|
188 |
if (consts.doExtendedLookup()) { |
|
189 |
alleDaekkedeAdresser = db.hentAlleDaekkedeAdresser(); |
alleDaekkedeAdresser = db.hentAlleDaekkedeAdresser(); |
190 |
logger.info( "AlleDaekkedeAdresser.length=" + alleDaekkedeAdresser.length); |
logger.info( "AlleDaekkedeAdresser.length=" + alleDaekkedeAdresser.size()); |
191 |
} |
|
192 |
|
short post = (short) 2700; |
193 |
Lookup lookup = new Lookup("458x", db, threadPool); |
Lookup lookup = new Lookup(post, db, threadPool); |
194 |
lookup.doLookup(); |
lookup.doLookup();*/ |
195 |
} |
} |
196 |
|
|
197 |
threadPool.shutdown(); |
|
|
|
|
198 |
long now = System.currentTimeMillis(); |
long now = System.currentTimeMillis(); |
199 |
long elapsed = now - start ; |
long elapsed = now - start ; |
200 |
|
long elapsedBeregn = now - beregnStart; |
201 |
logger.info("Mest brugte tid: " + mestBrugteTid); |
|
202 |
logger.info("Flest Ikke-dækkede, " + flestIkkeDaekkede); |
|
|
logger.info("Flest Dækkede, " + flestDaekkede); |
|
|
logger.info("Største Dataset, " + stoersteDataset); |
|
203 |
logger.info("Fuld load done : " + formatMilliSeconds(elapsed) ); |
logger.info("Fuld load done : " + formatMilliSeconds(elapsed) ); |
204 |
} |
logger.info("Fuld load done (beregning) : " + formatMilliSeconds(elapsedBeregn) ); |
205 |
|
logger.info("Antal daekkede : " + antalDaekkedeAdresser ); |
206 |
|
logger.info("Antal ikke-daekkede : " + antalIkkeDaekkede ); |
207 |
|
logger.info("Heraf, antal fundne : " + antalFundne ); |
208 |
|
|
209 |
|
logger.info( String.format("Fandt adresser til : %.2f %%", (antalFundne.get() *100.0)/antalIkkeDaekkede ) ); |
210 |
public static void saveStatistics(Statistik stat) { |
//logger.info("Fandt adresser til : " + (antalFundne*100.0)/ikkeDaekkede.length + "%" ); |
|
if (stat.antalDaekkede > flestDaekkede.antalDaekkede) { |
|
|
flestDaekkede = stat; |
|
|
} |
|
|
if (stat.antalIkkeDaekkede > flestIkkeDaekkede.antalIkkeDaekkede) { |
|
|
flestIkkeDaekkede = stat; |
|
|
} |
|
|
|
|
|
if (stat.totalDataset > mestBrugteTid.totalDataset) { |
|
|
stoersteDataset = stat; |
|
|
} |
|
|
|
|
|
if (stat.forbrugtTid > mestBrugteTid.forbrugtTid) { |
|
|
mestBrugteTid = stat; |
|
|
} |
|
211 |
} |
} |
212 |
|
|
213 |
|
|
214 |
|
|
215 |
static String formatMilliSeconds(long milliseconds) { |
static String formatMilliSeconds(long milliseconds) { |
216 |
int mseconds = (int) milliseconds % 1000; |
int mseconds = (int) milliseconds % 1000; |
217 |
int seconds = (int) (milliseconds / 1000) % 60 ; |
int seconds = (int) (milliseconds / 1000) % 60 ; |
218 |
int minutes = (int) ((milliseconds / (1000*60)) % 60); |
int minutes = (int) ((milliseconds / (1000*60)) % 60); |
219 |
int hours = (int) ((milliseconds / (1000*60*60)) % 24); |
int hours = (int) ((milliseconds / (1000*60*60)) % 24); |
220 |
|
|
221 |
return String.format("%02d:%02d:%02d.%03d", hours, minutes, seconds, mseconds); |
return String.format("%02d:%02d:%02d.%03d", hours, minutes, seconds, mseconds); |
222 |
} |
} |
223 |
|
|
224 |
static class WorkerThreadFactory implements ThreadFactory { |
|
225 |
int count = 0; |
static class WorkerThreadFactory implements ThreadFactory { |
226 |
|
int count = 0; |
227 |
@Override |
|
228 |
public Thread newThread(Runnable r) { |
@Override |
229 |
return new Thread(r, "lookupWorker/" + count++); |
public Thread newThread(Runnable r) { |
230 |
} |
return new Thread(r, "lookupWorker/" + count++); |
231 |
|
} |
232 |
} |
} |
233 |
} |
} |