View Javadoc
1   /**
2    * This Source Code Form is subject to the terms of the Mozilla Public
3    * License, v. 2.0. If a copy of the MPL was not distributed with this
4    * file, You can obtain one at http://mozilla.org/MPL/2.0/.
5    *
6    * If it is not possible or desirable to put the notice in a particular
7    * file, then You may include the notice in a location (such as a LICENSE
8    * file in a relevant directory) where a recipient would be likely to look
9    * for such a notice.
10   *
11   * 
12   */
13  /*  ---------------------------------------------------------------------------
14   *  U.S. Government, Department of the Army
15   *  Army Materiel Command
16   *  Research Development Engineering Command
17   *  Communications Electronics Research Development and Engineering Center
18   *  ---------------------------------------------------------------------------
19   */
20  package org.miloss.fgsms.bueller;
21  
22  import java.io.File;
23  import java.io.FileInputStream;
24  import java.io.InputStream;
25  import java.io.RandomAccessFile;
26  import java.net.HttpURLConnection;
27  import java.net.URL;
28  import java.nio.channels.FileChannel;
29  import java.nio.channels.FileLock;
30  import java.security.KeyStore;
31  import java.security.SecureRandom;
32  import java.sql.PreparedStatement;
33  import java.sql.ResultSet;
34  import java.sql.SQLException;
35  import java.util.ArrayList;
36  import java.util.Collections;
37  import java.util.List;
38  import java.util.Properties;
39  import javax.jms.*;
40  import javax.naming.Context;
41  import javax.naming.InitialContext;
42  import org.apache.http.HttpResponse;
43  import org.apache.http.HttpStatus;
44  import org.apache.http.auth.AuthScope;
45  import org.apache.http.auth.Credentials;
46  import org.apache.http.auth.NTCredentials;
47  import org.apache.http.auth.UsernamePasswordCredentials;
48  import org.apache.http.client.methods.HttpGet;
49  import org.apache.http.conn.scheme.Scheme;
50  import org.apache.http.conn.ssl.AllowAllHostnameVerifier;
51  import org.apache.http.conn.ssl.X509HostnameVerifier;
52  import org.apache.http.impl.client.DefaultHttpClient;
53  import org.apache.log4j.Level;
54  import org.miloss.fgsms.common.Logger;
55  ;
56  import org.apache.log4j.PropertyConfigurator;
57  import org.miloss.fgsms.common.DBSettingsLoader;
58  import org.miloss.fgsms.common.DBUtils;
59  import org.miloss.fgsms.common.Utility;
60  import org.miloss.fgsms.services.interfaces.common.PolicyType;
61  import org.miloss.fgsms.services.interfaces.policyconfiguration.KeyNameValueEnc;
62  import org.miloss.fgsms.services.interfaces.policyconfiguration.TransportAuthenticationStyle;
63  import org.miloss.fgsms.sla.AuxHelper;
64  import org.miloss.fgsms.sla.SLACommon;
65  
66  /**
67   * This tool for fgsms will fire off an HTTP GET request message in an attempt
68   * to retrieve the wsdl of a web service that is currently being monitored by
69   * fgsms. This will only function on services with an fgsms policy defined
70   * within the realm of the local instance of fgsms. It then set's the status of
71   * the service as known to fgsms. This information is available via the fgsms
72   * Status Service and is visually depicted within the fgsms GUI.
73   * Bueller.....Bueller.............Bueller..........................
74   *
75   * @author AO
76   */
77  
78  
79  public class Bueller {
80  
81      protected static final Logger log = Logger.getLogger("fgsms.StatusBueller");
82      public static boolean running = true;
83      public static boolean noloop = false;
84      private static String truststore = "";
85      private static String truststorepass = "";
86      private static String keystore = "";
87      private static String keystorepass = "";
88      private static long LastConfiguredAt = 0;
89      private static boolean Configured = false;
90      private static boolean ignoreSSL = false;
91      private static org.apache.http.conn.ssl.SSLSocketFactory sf = null;
92      private static org.apache.http.conn.ssl.SSLSocketFactory sfpki = null;
93      /**
94       * @param args the command line arguments
95       */
96      public static void main(String[] args) throws InterruptedException {
97          /*
98          * if (args.length == 1) { if (args[0].equalsIgnoreCase("start")) { // t
99          * = (new Thread(new DataPusher(policyCache, outboundQueue))); localref
100         * = new Bueller(); new Thread(localref).start();
101         *
102         * }
103         *
104         * if (args[0].equalsIgnoreCase("stop")) { running = false; } }
105         */
106         new Bueller().Main(args);
107         
108     }
109    
110     /**
111      * Alternate URLs, basically multiple hostnames, FQDN, host file entries, ip
112      * addresses, etc can map to the same service. Sometimes a particular url is
113      * firewalled or only listens on a specific hostname. By default bueller
114      * will first attempt a connection using the modified url, i.e. the URL
115      * displayed in fgsms which usually uses the hostname of the machine hosting
116      * the service If another url was observed by an agent at some point in
117      * time, then this will fetch all other urls for
118      *
119      * @param url
120      * @param perf
121      * @return
122      */
123     protected static List<String> GetAlternateUrls(String url, java.sql.Connection perf) {
124         List<String> alts = new ArrayList<String>();
125         PreparedStatement com = null;
126         ResultSet rs = null;
127         try {
128             
129             //dec 10-2011 PreparedStatement com = perf.prepareStatement("select  originalurl from rawdata where uri=? group by originalurl;");
130             com = perf.prepareStatement("select  alturi from alternateurls where uri=?;");
131             com.setString(1, url);
132             rs = com.executeQuery();
133             while (rs.next()) {
134                 String t = rs.getString(1);
135                 if (!Utility.stringIsNullOrEmpty(t)) {
136                     t = t.trim();
137                     if (!Utility.stringIsNullOrEmpty(t)) {
138                         if (!t.equals(url)) {
139                             if (!url.equals(t)) {
140                                 alts.add(t);
141                             }
142                         }
143                     }
144                 }
145                 //TODO future optimization but not required, this might be a good spot to filter out localhost/127.0.0.1 records, but only if this machine is different that the machine hosting the service
146             }
147         } catch (Exception ex) {
148             log.log(Level.ERROR, null, ex);
149         } finally {
150             DBUtils.safeClose(rs);
151             DBUtils.safeClose(com);
152         }
153         return alts;
154         
155     }
156     /**
157      * provides a wrapper for getting alternate urls without passing a db
158      * connection
159      *
160      * @param url
161      * @param pooled
162      * @return
163      */
164     protected static List<String> GetAlternateUrls(String url, boolean pooled) {
165         
166         java.sql.Connection perf = null;
167         if (pooled) {
168             perf = Utility.getPerformanceDBConnection();
169         } else {
170             perf = Utility.getPerformanceDB_NONPOOLED_Connection();
171         }
172         List<String> ret = Collections.EMPTY_LIST;
173         try {
174             ret = GetAlternateUrls(url, perf);
175         } catch (Exception ex) {
176             log.log(Level.ERROR, null, ex);
177         } finally {
178             DBUtils.safeClose(perf);
179         }
180         return ret;
181     }
182     private boolean done = false;
183     //~ Inner Classes -----------------------------------------------------------------------------
184     private File file;
185     private FileChannel channel;
186     private FileLock lock;
187 
188     private void closeLock() {
189         try {
190             lock.release();
191         } catch (Exception e) {
192         }
193         try {
194             channel.close();
195         } catch (Exception e) {
196         }
197     }
198 
199     private void deleteFile() {
200         try {
201             file.delete();
202         } catch (Exception e) {
203         }
204     }
205 
206     private void cleanUpOldStuff(List<String> MonitoredURLS, java.sql.Connection con) {
207         long timer = System.currentTimeMillis();
208         PreparedStatement cmd;
209         int x = 0;
210         try {
211             cmd = con.prepareStatement("delete from status where uri=(select status.uri from status  left outer join  servicepolicies  on (status.uri = servicepolicies.uri) where xmlpolicy is null)");
212             x = cmd.executeUpdate();
213         } catch (SQLException ex) {
214             log.log(Level.WARN, null, ex);
215         }
216 
217         if (x > 0) {
218             log.log(Level.INFO, "Purging " + x + " from status table");
219         }
220         log.log(Level.INFO, "Bueller cleanup performed in " + (System.currentTimeMillis() - timer) + "ms");
221         /*
222            * try { log.log(Level.INFO, "Performing cleanup operations");
223            * PreparedStatement com = con.prepareStatement("delete FROM status
224            * WHERE NOT EXISTS (SELECT * FROM servicepolicies WHERE " + "
225            * servicepolicies.policytype=? and servicepolicies.uri = status.uri and
226            * " + " servicepolicies.buellerenabled=true )"); com.setInt(1,
227            * PolicyType.TRANSACTIONAL.ordinal()); int i = com.executeUpdate(); if
228            * (i > 0) { log.log(Level.INFO, i + " records removed. This happens
229            * when I start running and I grab a list of services to ping and while
230            * executing, a service policy is deleted and we end up with an orphaned
231            * record."); } log.log(Level.INFO, "done.");
232            *
233            * } catch (SQLException ex) { log.log(Level.WARN, null, ex); }
234          */
235 
236     }
237 
238     public void run() {
239         this.Main(null);
240     }
241 
242     private String doJmsURL(boolean pooled, String endpoint) {
243         try {
244 
245             boolean ok = false;
246             String server = endpoint.split("#")[0];
247             server = server.replace("jms:", "jnp://");
248             String name = endpoint.split("#")[1];
249             String msg = "";
250             String[] info = DBSettingsLoader.GetCredentials(pooled, endpoint);
251             String username = null;
252             String password = null;
253             if (info != null) {
254                 username = info[0];
255                 password = info[1];
256             } else {
257                 info = DBSettingsLoader.GetDefaultBuellerCredentials(pooled);
258                 if (info != null) {
259                     username = info[0];
260                     password = info[1];
261                 }
262             }
263 
264             if (name.startsWith("topic")) {
265                 try {
266                     Properties properties1 = new Properties();
267                     properties1.put(Context.INITIAL_CONTEXT_FACTORY,
268                             "org.jnp.interfaces.NamingContextFactory");
269                     properties1.put(Context.URL_PKG_PREFIXES,
270                             "org.jboss.naming:org.jnp.interfaces");
271                     //properties1.put(Context.PROVIDER_URL, "jnp://127.0.0.1:1099");
272                     properties1.put(Context.PROVIDER_URL, server);
273 
274                     InitialContext iniCtx = new InitialContext(properties1);
275 
276                     TopicConnectionFactory tcf = (TopicConnectionFactory) iniCtx.lookup("TopicConnectionFactory");
277                     TopicConnection createTopicConnection = null;
278                     if (info != null) {
279                         createTopicConnection = tcf.createTopicConnection(username, Utility.DE(password)); //Topic topic = (Topic) iniCtx.lookup("/topic/quickstart_jmstopic_topic");
280                     } else {
281                         createTopicConnection = tcf.createTopicConnection(); //Topic topic = (Topic) iniCtx.lookup("/topic/quickstart_jmstopic_topic");
282                     }
283                     createTopicConnection.start();
284                     createTopicConnection.stop();
285                     createTopicConnection.close();
286                     //Topic topic = (Topic) iniCtx.lookup("//" + name);
287                     ok = true;
288 
289                     //topic = null;
290                     iniCtx.close();
291 
292                 } catch (Exception ex) {
293                     System.out.println(ex);
294                     msg = ex.getLocalizedMessage();
295                     //return ex.getLocalizedMessage();
296                 }
297             } else if (name.startsWith("queue")) {
298                 try {
299 
300                     Properties properties1 = new Properties();
301                     properties1.put(Context.INITIAL_CONTEXT_FACTORY,
302                             "org.jnp.interfaces.NamingContextFactory");
303                     properties1.put(Context.URL_PKG_PREFIXES,
304                             "org.jboss.naming:org.jnp.interfaces");
305                     properties1.put(Context.PROVIDER_URL, server);
306                     InitialContext iniCtx = new InitialContext(properties1);
307                     QueueConnection conn;
308                     QueueSession session;
309                     Queue que;
310 
311                     Object tmp = iniCtx.lookup("ConnectionFactory");
312                     QueueConnectionFactory qcf = (QueueConnectionFactory) tmp;
313                     if (info != null) {
314                         conn = qcf.createQueueConnection(username, Utility.DE(password));
315                     } else {
316                         conn = qcf.createQueueConnection();
317                     }
318 
319                     que = (Queue) iniCtx.lookup(name);
320                     session = conn.createQueueSession(false, QueueSession.AUTO_ACKNOWLEDGE);
321                     conn.start();
322 
323                     //System.out.println("Connection Started");
324                     ok = true;
325 
326                     conn.stop();
327                     session.close();
328                     iniCtx.close();
329 
330                 } catch (Exception ex) {
331                     log.log(Level.WARN, "Could not bind to jms queue", ex);
332                     msg = ex.getLocalizedMessage();
333                 }
334                 if (ok) {
335                     return "OK";
336                 }
337                 return "Unable to bind to JMS queue: " + msg;
338             } else {
339                 return "Unsupported Protocol";
340             }
341         } catch (Exception ex) {
342             log.log(Level.WARN, "service " + endpoint + " is offline or an error occured", ex);
343             return "Offline " + ex.getLocalizedMessage();
344         }
345         return "undeterminable";
346     }
347 
348 
349     private void Main(String args[]) {
350 
351         try {
352             file = new File("bueller.lck");
353             channel = new RandomAccessFile(file, "rw").getChannel();
354             lock = channel.tryLock();
355         } catch (Exception e) {
356             // already locked
357             closeLock();
358             System.out.println("Could not obtain the lock, this means that either this program is already running or something went wrong and the file bueller.lck needs to be deleted.");
359             return;
360         }
361         if (lock == null) {
362             closeLock();
363             System.out.println("Could not obtain the lock, this means that either this program is already running or something went wrong and the file bueller.lck needs to be deleted.");
364             return;
365         }
366 
367         Runtime.getRuntime().addShutdownHook(new RunWhenShuttingDown());
368 
369         PropertyConfigurator.configure("log4j.properties");
370 //        log.log(Level.INFO, "###########################################################");
371         log.log(Level.INFO, "fgsms.Bueller (service ping machine) startup.....");
372         //      log.log(Level.INFO, "###########################################################");
373         long lastRanAt = 0;
374         int interval = 10000;
375         if (args.length == 1 && args[0].equalsIgnoreCase("noloop")) {
376             noloop = true;
377         }
378 
379         while (running) {
380             if (System.currentTimeMillis() - lastRanAt > interval) {
381                 lastRanAt = System.currentTimeMillis();
382                 //run
383                 log.log(Level.INFO, "Bueller.....");
384                 try {
385                     Fire(false);
386 
387                     AuxHelper.TryUpdateStatus(true, "urn:fgsms:Bueller:" + Utility.getHostName(), "OK, took " + (System.currentTimeMillis() - lastRanAt) + "ms to process.", false, AuxHelper.UNSPECIFIED, SLACommon.GetHostName());
388 
389                 } catch (Exception ex) {
390                     //return;
391                     AuxHelper.TryUpdateStatus(false, "urn:fgsms:Bueller:" + Utility.getHostName(), "Running but possible database error" + (System.currentTimeMillis() - lastRanAt) + "ms to process.", false, AuxHelper.UNSPECIFIED, SLACommon.GetHostName());
392                     //commented out to that Bueller will continue to run, even if the database isn't available (the data will still be lost though)
393                 }
394                 log.log(Level.INFO, "Pausing until the next iteration in " + interval + "ms.....");
395 
396                 //          log.log(Level.INFO, "###########################################################");
397                 //         log.log(Level.INFO, "###########################################################");
398             }
399             if (noloop) {
400                 running = false;
401             }
402             if (running) {
403                 try {
404                     Thread.sleep(10000);
405                 } catch (InterruptedException ex) {
406                     log.log(Level.FATAL, null, ex);
407                 }
408             }
409 
410         }
411         done = true;
412     }
413 
414 
415     protected void Init(boolean pooled) throws Exception {
416         if ((System.currentTimeMillis() - 30000) < LastConfiguredAt && Configured) {
417             log.log(Level.DEBUG, "already configured, using last known config");
418             return;
419         }
420         log.log(Level.INFO, "Refreshing configuration");
421         LastConfiguredAt = System.currentTimeMillis();
422         Configured = true;
423 
424         String tmp = System.getProperty("jboss.server.config.url");
425 
426         if (Utility.stringIsNullOrEmpty(tmp)) {
427             //FIX for Jboss 7
428             try {
429                 tmp = System.getProperty("jboss.server.config.dir");
430                 if (tmp != null && !tmp.equalsIgnoreCase("null")) {
431                     File f = new File(tmp);
432                     tmp = f.toURI().toURL().toString();
433                     tmp += File.separator;
434                 }
435             } catch (Exception e) {
436             }
437         }
438         //fix for tomcat
439         if (Utility.stringIsNullOrEmpty(tmp)) {
440             tmp = System.getProperty("catalina.home");
441             if (tmp != null) {
442                 tmp = tmp + File.separator + "conf" + File.separator;
443             }
444         }
445         //fix for OpenJDK/linux issues
446         if (tmp != null) {
447             String os = System.getProperty("os.name");
448             if (os.toLowerCase().indexOf("win") == -1) {
449                 tmp = File.separator + tmp;
450             }
451 
452         }
453         X509HostnameVerifier hostnameVerifier = new org.apache.http.conn.ssl.StrictHostnameVerifier();
454         KeyNameValueEnc d = DBSettingsLoader.GetPropertiesFromDB(pooled, "Bueller", "IgnoreSSLErrors");
455         if (d != null && d.getKeyNameValue() != null) {
456             try {
457                 ignoreSSL = Boolean.parseBoolean(d.getKeyNameValue().getPropertyValue());
458             } catch (Exception ex) {
459                 ignoreSSL = false;
460             }
461         }
462         if (ignoreSSL) {
463             log.log(Level.WARN, "SSL Hostname verification turned off");
464             hostnameVerifier = new AllowAllHostnameVerifier();
465         }
466 
467         if (!Utility.stringIsNullOrEmpty(tmp)) {
468 
469             d = DBSettingsLoader.GetPropertiesFromDB(pooled, "defaults", "truststore");
470             if (d != null && d.getKeyNameValue() != null) {
471                 truststore = tmp + d.getKeyNameValue().getPropertyValue();
472             }
473             d = DBSettingsLoader.GetPropertiesFromDB(pooled, "defaults", "truststorepass");
474             if (d != null && d.getKeyNameValue() != null) {
475                 truststorepass = d.getKeyNameValue().getPropertyValue();
476             }
477             d = DBSettingsLoader.GetPropertiesFromDB(pooled, "defaults", "keystore");
478             if (d != null && d.getKeyNameValue() != null) {
479                 keystore = tmp + d.getKeyNameValue().getPropertyValue();
480             }
481             d = DBSettingsLoader.GetPropertiesFromDB(pooled, "defaults", "keystorepass");
482             if (d != null && d.getKeyNameValue() != null) {
483                 keystorepass = d.getKeyNameValue().getPropertyValue();
484             }
485 
486             if (!(Utility.stringIsNullOrEmpty(truststore) && !Utility.stringIsNullOrEmpty(truststorepass)) && !Utility.stringIsNullOrEmpty(keystore) && !Utility.stringIsNullOrEmpty(keystorepass)) {
487 
488                 KeyStore trustStore = null;
489                 FileInputStream instream = null;
490                 try {
491                     log.log(Level.INFO, "loading trust store from " + truststore);
492                     instream = new FileInputStream(new File(truststore.replace("file:/", "")));
493                     trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
494                     trustStore.load(instream, Utility.DE(truststorepass).toCharArray());
495                 } catch (Exception ex) {
496                     log.log(Level.WARN, null, ex);
497                     trustStore = null;
498                 } finally {
499                     if (instream != null) {
500                         try {
501                             instream.close();
502                         } catch (Exception ex) {
503                             log.log(Level.DEBUG, null, ex);
504                         }
505                     }
506                 }
507                 KeyStore keyStore = null;
508 
509                 try {
510                     log.log(Level.INFO, "loading key store from " + keystore);
511                     instream = new FileInputStream(new File(keystore.replace("file:/", "")));
512                     keyStore = KeyStore.getInstance(KeyStore.getDefaultType());
513                     keyStore.load(instream, Utility.DE(keystorepass).toCharArray());
514                 } catch (Exception ex) {
515                     log.log(Level.WARN, null, ex);
516                     keyStore = null;
517                 } finally {
518                     if (instream != null) {
519                         try {
520                             instream.close();
521                         } catch (Exception ex) {
522                             log.log(Level.DEBUG, null, ex);
523                         }
524                     }
525                 }
526                 if (trustStore != null && keyStore != null) {
527                     sfpki = new org.apache.http.conn.ssl.SSLSocketFactory(org.apache.http.conn.ssl.SSLSocketFactory.TLS, keyStore, Utility.DE(keystorepass), trustStore, new SecureRandom(),
528                             hostnameVerifier);
529                     //sf = new org.apache.http.conn.ssl.SSLSocketFactory(asdasdtrustStore, Utility.DE(truststorepass));
530                 } else if (trustStore != null) {
531                     sf = new org.apache.http.conn.ssl.SSLSocketFactory(trustStore, Utility.DE(truststorepass));
532                 }
533             } else if (!(Utility.stringIsNullOrEmpty(truststore) && !Utility.stringIsNullOrEmpty(truststorepass))) {
534                 KeyStore trustStore = null;
535                 FileInputStream instream = new FileInputStream(new File(truststore.replace("file:/", "")));
536                 try {
537                     log.log(Level.INFO, "loading trust store from " + truststore);
538                     trustStore = KeyStore.getInstance(KeyStore.getDefaultType());
539                     trustStore.load(instream, Utility.DE(truststorepass).toCharArray());
540                 } catch (Exception ex) {
541                     log.log(Level.WARN, null, ex);
542                     trustStore = null;
543                 } finally {
544                     if (instream != null) {
545                         try {
546                             instream.close();
547                         } catch (Exception ex) {
548                             log.log(Level.DEBUG, null, ex);
549                         }
550                     }
551                 }
552                 if (trustStore != null) {
553                     sf = new org.apache.http.conn.ssl.SSLSocketFactory(trustStore, Utility.DE(truststorepass));
554                     sf.setHostnameVerifier(hostnameVerifier);
555                 }
556             }
557 
558             //check system.properties for javax.net.ssl
559             //load trust store and keystore from jboss?
560             //setup ssl 
561         } else {
562             log.log(Level.WARN, "unable to determine the location to the key/trust stores because the environment variable jboss.server.config.url, jboss.server.config.dir and catalina.home are not defined. At least one must be set for SSL to function");
563         }
564     }
565 
566     /**
567      * Performs the work of status bueller by sending an http get message to
568      * attempt to pull the wsdl Loads all service urls that are of PolicyType
569      * transactional
570      *
571      * @param pooled
572      * @throws SQLException
573      * @throws Exception
574      */
575     public void Fire(boolean pooled) throws SQLException, Exception {
576         Init(pooled);
577         java.sql.Connection con = null;
578         java.sql.Connection perf = null;
579         if (pooled) {
580             con = Utility.getConfigurationDBConnection();
581             perf = Utility.getPerformanceDBConnection();
582         } else {
583             con = Utility.getConfigurationDB_NONPOOLED_Connection();
584             perf = Utility.getPerformanceDB_NONPOOLED_Connection();
585         }
586         PreparedStatement com = null;
587         ResultSet rs = null;
588         List<String> urls = new ArrayList<String>();
589         try {
590             boolean run = true;
591             KeyNameValueEnc GetPropertiesFromDB = DBSettingsLoader.GetPropertiesFromDB(pooled, "Bueller", "Enabled");
592             if (GetPropertiesFromDB != null && GetPropertiesFromDB.getKeyNameValue() != null) {
593                 try {
594                     run = Boolean.parseBoolean(GetPropertiesFromDB.getKeyNameValue().getPropertyValue());
595                 } catch (Exception ex) {
596                 }
597             }
598             if (!run) {
599                 log.log(Level.INFO, "Bueller is disabled by General Settings, key=Bueller, name=Enabled, exiting...");
600                 con.close();
601                 perf.close();
602                 return;
603             }
604 
605             //snip, there used to be code here to tie into a jboss workmanager was it was removed due to licensing concerns.
606             com = con.prepareStatement("select uri from servicepolicies where buellerenabled=true and policytype=?; ");
607             com.setInt(1, PolicyType.TRANSACTIONAL.ordinal());
608             rs = com.executeQuery();
609 
610             while (rs.next()) {
611                 urls.add(rs.getString(1));
612             }
613             rs.close();
614             com.close();
615         } catch (Exception ex) {
616             log.log(Level.ERROR, "unexpected error running bueller ", ex);
617 
618         } finally {
619             DBUtils.safeClose(rs);
620             DBUtils.safeClose(com);
621         }
622 
623         try {
624             log.log(Level.INFO, urls.size() + " urls to process");
625 
626             {
627                 for (int i = 0; i < urls.size(); i++) {
628                     if (!running) {
629                         log.log(Level.INFO, "Interupt detected on url " + (i + 1) + " of " + urls.size());
630                         break;
631                     }
632                     try {
633                         String s = sendGetRequest(pooled, urls.get(i), 0);
634                         Boolean currenstatus = s.equalsIgnoreCase("ok");
635                         log.log(Level.INFO, (i + 1) + "/" + urls.size() + " " + urls.get(i) + " status is " + s);
636                         if (!currenstatus) {
637                             List<String> alts = GetAlternateUrls(urls.get(i), perf);
638                             for (int k = 0; k < alts.size(); k++) {
639                                 if (currenstatus) {
640                                     break;
641                                 }
642                                 s = sendGetRequest(pooled, alts.get(k), 0);
643                                 currenstatus = s.equalsIgnoreCase("ok");
644                                 log.log(Level.INFO, urls.get(i) + " via alternate URL " + alts.get(k) + " status is " + s);
645                             }
646                         }
647 
648                         AuxHelper.TryUpdateStatus(currenstatus, urls.get(i), s, pooled, AuxHelper.UNSPECIFIED, SLACommon.GetHostName(), AuxHelper.FLAGS.NO_AUTO_CREATE);
649 
650                     } catch (Exception ex) {
651                         log.log(Level.ERROR, "error setting status in config db for uri " + urls.get(i), ex);
652                     }
653                 }
654 
655                 cleanUpOldStuff(urls, con);
656             }
657         } catch (Exception ex) {
658             log.log(Level.ERROR, "unexpected error running bueller ", ex);
659 
660         } finally {
661             DBUtils.safeClose(con);
662             DBUtils.safeClose(perf);
663         }
664     }
665 
666     protected Credentials transformCredentials(String[] info) {
667         if (info.length == 2) {
668             return new UsernamePasswordCredentials(info[0], Utility.DE(info[1]));
669         }
670         if (info.length == 3) {
671             TransportAuthenticationStyle tas = TransportAuthenticationStyle.valueOf(info[2]);
672             switch (tas) {
673                 case HTTP_NTLM:
674                     String data = info[0];
675                     String[] t = data.split("\\\\");
676                     String username = t[1];
677                     String domain = t[0];
678                     return new NTCredentials(username, Utility.DE(info[1]), SLACommon.GetHostName(), domain);
679                 case HTTP_DIGEST:
680                 case HTTP_BASIC:
681                 default:
682                     return new UsernamePasswordCredentials(info[0], Utility.DE(info[1]));
683             }
684         }
685         return null;
686     }
687 
688     /**
689      * attempts an http get request with authentication
690      *
691      * @param pooled
692      * @param endpoint
693      * @param policyURL
694      * @return
695      */
696     protected String sendGetRequestAuth(boolean pooled, String endpoint, String policyURL, int depth) {//, AuthMode mode) {
697         if (depth > 10) {
698             //abort, possible redirect loop
699             return "Aborting due to redirect loop";
700         }
701         String[] info = DBSettingsLoader.GetCredentials(pooled, policyURL);
702 
703         if (info == null) {
704             info = DBSettingsLoader.GetDefaultBuellerCredentials(pooled);
705             if (info == null) {
706                 return "Unauthorized, no credentials are available";
707             }
708         }
709 
710         if (endpoint.startsWith("http://")) {
711             // Send a GET request to the servlet
712 
713             DefaultHttpClient c = new DefaultHttpClient();
714             try {
715                 c.getCredentialsProvider().setCredentials(AuthScope.ANY, transformCredentials(info));
716                 if (!c.getCredentialsProvider().getCredentials(AuthScope.ANY).getClass().getCanonicalName().equalsIgnoreCase(NTCredentials.class.getCanonicalName())) {
717                     log.log(Level.WARN, "Usage of non-NTLM authentication over a non SSL channel.");
718                 }
719                 HttpGet m = new HttpGet(endpoint);
720                 HttpResponse res = c.execute(m);
721                 int status = res.getStatusLine().getStatusCode();
722                 try {
723                     InputStream content = res.getEntity().getContent();
724                     byte[] buffer = new byte[1024];
725                     while (content.read(buffer) >= 0) {
726                     }
727                 } catch (Exception f) {
728                 }
729                 c.getConnectionManager().shutdown();
730                 if (status < 300 || status == HttpStatus.SC_NOT_MODIFIED) {
731                     return "OK";
732                 } else if (status == HttpStatus.SC_MOVED_PERMANENTLY
733                         || status == HttpStatus.SC_TEMPORARY_REDIRECT
734                         || status == HttpStatus.SC_MOVED_TEMPORARILY) {
735                     String newUrl = res.getHeaders("Location")[0].getValue();
736                     return sendGetRequestAuth(pooled, newUrl, policyURL, depth + 1);
737                 }
738 
739                 return String.valueOf(status);
740             } catch (Exception ex) {
741                 c.getConnectionManager().shutdown();
742                 log.log(Level.INFO, "code " + ex.getLocalizedMessage());
743                 return "offline: " + ex.getMessage();
744             }
745         } else if (endpoint.startsWith("https://")) {
746 
747             //first try with the username/password over ssl
748             if (sf == null && sfpki == null) {
749                 return "Unauthorized, no trust store available for SSL communication";
750             }
751             DefaultHttpClient c = new DefaultHttpClient();
752             try {
753                 URL url = new URL(endpoint);
754                 String scheme = "https";
755                 int port = url.getPort();
756 
757                 if (port == -1 && endpoint.toLowerCase().startsWith("https:")) {
758                     port = 443;
759                 }
760 
761                 Scheme sch = null;
762 
763                 if (sfpki == null) {
764                     sch = new Scheme("https", port, sf);
765                 } else {
766                     sch = new Scheme("https", port, sfpki);
767                 }
768                 c.getConnectionManager().getSchemeRegistry().register(sch);
769 
770                 c.getCredentialsProvider().setCredentials(AuthScope.ANY, transformCredentials(info));
771 
772                 HttpGet m = new HttpGet(endpoint);
773                 HttpResponse res = c.execute(m);
774                 int status = res.getStatusLine().getStatusCode();
775                 try {
776                     InputStream content = res.getEntity().getContent();
777                     byte[] buffer = new byte[1024];
778                     while (content.read(buffer) >= 0) {
779                     }
780                 } catch (Exception f) {
781                 }
782                 c.getConnectionManager().shutdown();
783                 if (status < 300 || status == HttpStatus.SC_NOT_MODIFIED) {
784                     return "OK";
785                 } else if (status == HttpStatus.SC_MOVED_PERMANENTLY
786                         || status == HttpStatus.SC_TEMPORARY_REDIRECT
787                         || status == HttpStatus.SC_MOVED_TEMPORARILY) {
788                     String newUrl = res.getHeaders("Location")[0].getValue();
789                     return sendGetRequestAuth(pooled, newUrl, policyURL, depth + 1);
790                 }
791 
792                 return String.valueOf(status);
793             } catch (Exception ex) {
794                 c.getConnectionManager().shutdown();
795                 log.log(Level.INFO, "code " + ex.getLocalizedMessage());
796                 return "offline: " + ex.getMessage();
797             }
798 
799         } else {
800             return "undeterminable";
801         }
802 
803     }
804 
805 
806     /**
807      * Sends an HTTP GET request to a url
808      *
809      * @param endpoint - The URL of the server. (Example: "
810      * http://www.yahoo.com/search") Note: This method will add the question
811      * mark (?wsdl) to the request
812      * @return - OK for 200 messages, all others, the actually response code or
813      * error message
814      */
815     protected String sendGetRequest(boolean pooled, String endpoint, int depth) {
816         if (depth > 10) {
817             //abort, possible redirect loop
818             return "Aborting due to redirect loop";
819         }
820         String result = null;
821         String policyUrl = new String(endpoint);
822         if (endpoint.startsWith("http://")) {
823             // Send a GET request to the servlet
824             HttpURLConnection conn = null;
825             try {
826                 String originalendpoint = endpoint;
827                 if (!endpoint.endsWith("?wsdl")) {
828                     endpoint = endpoint + "?wsdl";
829                 }
830                 conn = (HttpURLConnection) new URL(endpoint).openConnection();
831 
832                 if (conn.getResponseCode() == 401) {
833                     //basic example WWW-Authenticate: Basic realm="fgsms Services"
834                     //digest example WWW-Authenticate: Digest realm="fgsms Services", qop="auth", nonce="2569aa2af54f6d47e8472f47f2e3da01", opaque="a39d25cce80574f8255b97052d8f1544"
835                     //WWW-Authenticate: Negotiate
836                     //WWW-Authenticate: NTLM
837                     //        String authtype = conn.getHeaderField("WWW-Authenticate");
838                     //    if (authtype.toLowerCase().startsWith("digest")) {
839                     return sendGetRequestAuth(pooled, endpoint, policyUrl, depth + 1);
840                 } else if (conn.getResponseCode() == 404) {
841                     //fix for sonatype nexus and non wsdl urls
842                     conn = (HttpURLConnection) new URL(originalendpoint).openConnection();
843                     return "Not found";
844                 } else if (conn.getResponseCode() == HttpStatus.SC_MOVED_PERMANENTLY
845                         || conn.getResponseCode() == HttpStatus.SC_MOVED_TEMPORARILY
846                         || conn.getResponseCode() == HttpStatus.SC_TEMPORARY_REDIRECT) {
847                     //follow the redirect
848                     String newUrl = conn.getHeaderField("Location");
849                     return sendGetRequest(pooled, newUrl, depth + 1);
850                     //System.out.println("Moved to " + newUrl); //should be the new destination url
851                     //return "Moved " + conn.getResponseMessage();
852                 } else if (conn.getResponseCode() == HttpStatus.SC_NOT_MODIFIED) {
853                     return "OK";
854                 }
855                 InputStream inputStream = null;
856                 try {
857                     inputStream = conn.getInputStream();
858                     byte[] buffer = new byte[1024];
859                     while (inputStream.read(buffer) >= 0) {
860                     }
861                     inputStream.close();
862                 } catch (Exception f) {
863                 } finally {
864                     if (inputStream != null) {
865                         try {
866                             inputStream.close();
867                         } catch (Exception ex) {
868                         }
869                     }
870                 }
871 
872                 String msg = conn.getResponseMessage();
873                 conn.disconnect();
874                 return msg;
875             } catch (java.net.UnknownHostException ex) {
876                 return "Host unknown";
877             } catch (Exception ex) {
878                 return ex.getMessage();
879             } finally {
880                 if (conn != null) {
881                     try {
882                         conn.disconnect();
883                     } catch (Exception ex) {
884                     }
885                 }
886             }
887         } else if (endpoint.startsWith("https://")) {
888             if (!endpoint.endsWith("?wsdl")) {
889                 endpoint = endpoint + "?wsdl";
890             }
891             DefaultHttpClient c = new DefaultHttpClient();
892             try {
893                 URL url = new URL(endpoint);
894                 int port = url.getPort();
895                 if (port == -1 && endpoint.toLowerCase().startsWith("http:")) {
896                     port = 80;
897                 }
898                 if (port == -1 && endpoint.toLowerCase().startsWith("https:")) {
899                     port = 443;
900                 }
901                 Scheme sch = null;
902                 if (sfpki == null) {
903                     sch = new Scheme("https", port, sf);
904                 } else {
905                     sch = new Scheme("https", port, sfpki);
906                 }
907                 if (endpoint.toLowerCase().startsWith("https:")) {
908                     c.getConnectionManager().getSchemeRegistry().register(sch);
909                 }
910 
911                 HttpGet m = new HttpGet(endpoint);
912                 HttpResponse res = c.execute(m);
913                 int status = res.getStatusLine().getStatusCode();
914                 try {
915                     InputStream content = res.getEntity().getContent();
916                     byte[] buffer = new byte[1024];
917                     while (content.read(buffer) >= 0) {
918                     }
919                 } catch (Exception f) {
920                 }
921                 c.getConnectionManager().shutdown();
922                 if (status == 401) {
923                     return sendGetRequestAuth(pooled, endpoint, policyUrl, depth + 1);
924                 } else if (status == HttpStatus.SC_MOVED_PERMANENTLY
925                         || status == HttpStatus.SC_MOVED_TEMPORARILY
926                         || status == HttpStatus.SC_TEMPORARY_REDIRECT) {
927                     String newUrl = res.getHeaders("Location")[0].getValue();
928                     return sendGetRequest(pooled, newUrl, depth + 1);
929                 } else if (status == HttpStatus.SC_NOT_MODIFIED) {
930                     return "OK";
931                 } else {
932                     return (status < 300) ? "OK" : "offline";
933                 }
934             } catch (Exception ex) {
935                 c.getConnectionManager().shutdown();
936                 log.log(Level.WARN, "error caught connecting to " + endpoint, ex);
937                 return ex.getMessage();
938             }
939 
940         } else if (endpoint.startsWith("jms:")) {
941             return doJmsURL(pooled, endpoint);
942         }
943         return "Unknown protocol";
944     }
945 
946     public class RunWhenShuttingDown extends Thread {
947         
948         public void run() {
949             System.out.println("Control-C caught. Shutting down...");
950             running = false;
951             while (!done) {
952                 try {
953                     Thread.sleep(1000);
954                 } catch (InterruptedException ex) {
955                 }
956             }
957             closeLock();
958             deleteFile();
959             if (!noloop) {
960                 AuxHelper.TryUpdateStatus(false, "urn:fgsms:Bueller:" + Utility.getHostName(), "Stopped", false, AuxHelper.UNSPECIFIED, SLACommon.GetHostName());
961             }
962         }
963     }
964 }