--- dao/FuldDaekningWorker/src/dk/daoas/fulddaekning/LookupMain.java	2015/06/14 15:15:00	2589
+++ dao/FuldDaekningWorker/src/dk/daoas/fulddaekning/LookupMain.java	2015/09/27 20:00:46	2714
@@ -1,163 +1,176 @@
 package dk.daoas.fulddaekning;
 
-import geocode.GeoPoint;
-import geocode.kdtree.KDTree;
-
 import java.io.File;
 import java.io.FileReader;
-import java.sql.SQLException;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
+import java.util.Queue;
 import java.util.Set;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.logging.FileHandler;
-import java.util.logging.Level;
 import java.util.logging.Logger;
 import java.util.logging.SimpleFormatter;
 
+import com.jwetherell.algorithms.data_structures.KdTree;
+
+
 public class LookupMain {
-	
-	 static class LookupResult {
-		 double bedsteAfstand = 9999999.999;
-		 Adresse bedsteAdr = null;
-	 }
 
 
 	static final String CONFIG_FILENAME = "fulddaekning.properties";
-	
+
 	static boolean rename_tables;
 	static SafeProperties conf;
-	
+
 	static int max_workers;
 	static boolean verbose;
-	
+
 	static String distributor;
 
 	final static Logger logger = Logger.getLogger( LookupMain.class.toString() );
-	
-	
-	
-	private static List<GeoPoint> alleDaekkedeAdresser;
 
-	
-	
+
+	static ThreadPoolExecutor threadPool;
+
+	private static void setupThreadPool() {	 
+		threadPool = (ThreadPoolExecutor) Executors.newFixedThreadPool(max_workers, new WorkerThreadFactory() );	 
+	}
+
+
+
 	public static void main(String[] args) throws Exception {
-		
+
 		//Setup j.u.l Logger
 		Logger root = Logger.getLogger("");
 		FileHandler fhandler = new FileHandler("fulddaekning.log"); // Ingen max størrelse, ingen rotation og ingen append
 		fhandler.setFormatter( new SimpleFormatter() );
 		root.addHandler( fhandler );
-		
-		
-		
-		
+
+
+
+
 		File confFile = new File( CONFIG_FILENAME );
 		if (! confFile.exists() ) {
 			logger.warning("Config file not found: " + CONFIG_FILENAME);
 			System.exit(1);
 		}
-				
+
 		conf = new SafeProperties();
 		conf.load( new FileReader(confFile) );
-		
-		
-		
+
+
+
 		verbose = Boolean.parseBoolean( conf.getSafeProperty("VERBOSE") );
 		logger.info("Starting with VERBOSE:" + verbose);
-		
+
 		rename_tables = Boolean.parseBoolean( conf.getSafeProperty("RENAMETABLES") );
 		logger.info("Starting with RENAMETABLES:" + rename_tables);
-		
+
 		distributor = conf.getSafeProperty("DISTRIBUTOR");
 		distributor = distributor.toUpperCase();
 		logger.info("Starting for DISTRIBUTOR:" + distributor);
-		
+
 		Constants.init(distributor);
 		Constants consts = Constants.getInstance();
-		
+
 		Database db = new Database(conf);
-		Adresse ikkeDaekkede[] = db.hentAlleIkkedaekkedeAdresser(consts.getMinPostnr(), consts.getMaxPostnr() );
+		Queue<Adresse> ikkeDaekkede = db.hentAlleIkkedaekkedeAdresser(consts.getMinPostnr(), consts.getMaxPostnr() );
+		logger.info("Antal ikke-daekkede: " + ikkeDaekkede.size() );
+
+
+		max_workers = Integer.parseInt( conf.getSafeProperty("MAX_WORKERS") );	 
+		if (max_workers <= 0) {	 
+			logger.info("!!! AUTO-DETECT MAX_WORKERS !!!");	 
+			int cores = Runtime.getRuntime().availableProcessors();	 
+			cores -= 1;//Efterlad 1 core/cpu i reserve til systemet	 
+
+			max_workers = Math.max(1, cores); //Dog skal der som minimum være 1 core til beregning	 
+
+		}	 
+		logger.info("Starting with MAX_WORKERS:" + max_workers);	 
+		setupThreadPool();
 
-		
 		boolean testRun= false;
-		
-		int antalFundne = 0;
-		
-		
+
+		AtomicInteger antalFundne = new AtomicInteger(0);
+
+		long beregnStart =0;
 		long start = System.currentTimeMillis();
-		
+		int antalDaekkedeAdresser = -1;
+
 		if (testRun == false) {
-		
+
 			logger.info("Finder postnumre");
 			Set<Short> postnumre = db.hentPostnumreCache();
-			
+
 			// Først validerer vi BBox på alle postnummre, for at undgå fuldt stop midt i beregningen
 			for(short postnr : postnumre) { //
 				logger.info("Validerer BBox for " + postnr);
 				BoundingBox bbox = db.getBoundingbox(postnr);
 				bbox.validateBbox();
 			}
-			
+
 
 			logger.info("Henter alle daekkede adresser");
-			alleDaekkedeAdresser = db.hentAlleDaekkedeAdresser();
-			logger.info( "AlleDaekkedeAdresser.length=" + alleDaekkedeAdresser.size());
-			
-			KDTree<GeoPoint> addressTree = new KDTree<GeoPoint>(alleDaekkedeAdresser);
-			
+			antalDaekkedeAdresser = db.hentAlleDaekkedeAdresser();
+			logger.info( "AlleDaekkedeAdresser.length=" + antalDaekkedeAdresser );
+
+			Map<Short, List<Adresse>> addrHoList = db.getDaekkedeAdresserHO();
+
+			Map<Short, KdTree<Adresse>> hoTrees = new HashMap<Short,KdTree<Adresse>>();
+
+			for ( Map.Entry<Short, List<Adresse>> entry : addrHoList.entrySet() ) {
+				short ho = entry.getKey();
+				List<Adresse> geopoints = entry.getValue();
+
+				logger.info("Opbygger KDTree for " + ho + " - antal=" + geopoints.size() );
+
+				@SuppressWarnings("unchecked")
+				List<KdTree.XYZPoint> tmpPoints = (List<KdTree.XYZPoint>)(List<?>) geopoints;
+
+				KdTree<Adresse> addressTree = new KdTree<Adresse>( tmpPoints );
+				hoTrees.put(ho, addressTree);				
+			}
+
 			db.resetResultTable();
-		
+
+			beregnStart = System.currentTimeMillis();
 			logger.info("Starter beregning");
-		
-			final int workerId = -1;
+
 			//pre-check er ok - reset tmp tabel og start søgningen
 			
-			for (Adresse qAdresse : ikkeDaekkede) {
-				
-				
-				LookupResult result = new LookupResult();
-				result.bedsteAdr = (Adresse) addressTree.findNearest(qAdresse);
-				result.bedsteAfstand = GeoPointHelper.beregnAfstand(qAdresse, result.bedsteAdr);
+			CyclicBarrier barrier = new CyclicBarrier(LookupMain.max_workers + 1);
 			
-			
-			
-				if (result.bedsteAfstand <= consts.getMaxDistance() ) { //Gem kun hvis distancen er under max_distance for distributøren
-					if (verbose == true) {
-						logger.fine("(" +workerId + ") Bedste: " + qAdresse + "==>" + result.bedsteAdr + " ::: " + result.bedsteAfstand );
-					}
-					
-					antalFundne++;
-					
-					try {
-						db.gemResultat(qAdresse, result.bedsteAdr, result.bedsteAfstand);
-					} catch (SQLException e) {
-						logger.log(Level.WARNING, "Error saving result", e);
-						logger.warning("Shutting down");
-						
-						System.exit(0); //hvis at indsættelsen fejler skal vi afbryde med det samme
-					}
-				} else {
-					if (verbose == true) {
-						System.out.println("For stor afstand: " + qAdresse + " > " + result.bedsteAfstand);
-					}
-				}
-				
+			for (int i=0; i<max_workers; i++) {
+				LookupWorker worker = new LookupWorker(i, barrier, ikkeDaekkede, hoTrees,antalFundne,db,verbose,consts);
+				threadPool.submit( worker );
 			}
 			
-			db.saveBatch();
+			barrier.await(); // Afvent at workerne bliver færdige
 			
+			threadPool.shutdown(); //Calc is done now
+
+
+			db.saveBatch();
+
 			if (rename_tables) {
 				db.renameResultTables();
 			} else {
 				logger.info( "Rename tables is disabled !!!" );
 			}
-			
-	
+
+
 
 		} else {
 			/// Test
 			/*
 			db.resetResultTable();
-			
+
 
 			alleDaekkedeAdresser = db.hentAlleDaekkedeAdresser();
 			logger.info( "AlleDaekkedeAdresser.length=" + alleDaekkedeAdresser.size());
@@ -167,27 +180,40 @@
 			lookup.doLookup();*/								
 		}
 
-		
+
 		long now = System.currentTimeMillis();
 		long elapsed = now - start ;
-		
+		long elapsedBeregn = now - beregnStart;
+
 
 		logger.info("Fuld load done : " + formatMilliSeconds(elapsed) );
-		logger.info("Antal daekkede : " + alleDaekkedeAdresser.size() );
-		logger.info("Antal ikke-daekkede : " + ikkeDaekkede.length );
+		logger.info("Fuld load done (beregning) : " + formatMilliSeconds(elapsedBeregn) );
+		logger.info("Antal daekkede : " + antalDaekkedeAdresser );
+		logger.info("Antal ikke-daekkede : " + ikkeDaekkede.size() );
 		logger.info("Heraf, antal fundne : " + antalFundne );
-		logger.info("Fandt adresser til : " + (antalFundne*100.0)/ikkeDaekkede.length + "%" );
+
+		logger.info( String.format("Fandt adresser til : %.2f %%", (antalFundne.get() *100.0)/ikkeDaekkede.size() ) );
+		//logger.info("Fandt adresser til : " + (antalFundne*100.0)/ikkeDaekkede.length + "%" );
 	}
 
-	
+
 
 	static String formatMilliSeconds(long milliseconds) {
 		int mseconds = (int) milliseconds % 1000;
 		int seconds = (int) (milliseconds / 1000) % 60 ;
 		int minutes = (int) ((milliseconds / (1000*60)) % 60);
 		int hours   = (int) ((milliseconds / (1000*60*60)) % 24);
-		
+
 		return String.format("%02d:%02d:%02d.%03d", hours, minutes, seconds, mseconds);
 	}
-	
+
+
+	static class WorkerThreadFactory implements ThreadFactory {	 
+		int count = 0;	 
+
+		@Override	 
+		public Thread newThread(Runnable r) {	 
+			return new Thread(r, "lookupWorker/" + count++);	 
+		}
+	}
 }