View Javadoc
1   /**
2    * This Source Code Form is subject to the terms of the Mozilla Public
3    * License, v. 2.0. If a copy of the MPL was not distributed with this
4    * file, You can obtain one at http://mozilla.org/MPL/2.0/.
5    *
6    * If it is not possible or desirable to put the notice in a particular
7    * file, then You may include the notice in a location (such as a LICENSE
8    * file in a relevant directory) where a recipient would be likely to look
9    * for such a notice.
10   *
11   * 
12   */
13  /*  ---------------------------------------------------------------------------
14   *  U.S. Government, Department of the Army
15   *  Army Materiel Command
16   *  Research Development Engineering Command
17   *  Communications Electronics Research Development and Engineering Center
18   *  ---------------------------------------------------------------------------
19   */
20  package org.miloss.fgsms.services.dcs.impl;
21  
22  import java.io.ByteArrayInputStream;
23  import java.net.InetAddress;
24  import java.sql.Connection;
25  import java.sql.PreparedStatement;
26  import java.sql.ResultSet;
27  import java.sql.SQLException;
28  import java.util.GregorianCalendar;
29  import java.util.HashMap;
30  import java.util.List;
31  import java.util.UUID;
32  import javax.annotation.Resource;
33  import javax.jws.WebMethod;
34  import javax.jws.WebParam;
35  import javax.jws.WebResult;
36  import javax.jws.WebService;
37  import javax.jws.soap.SOAPBinding;
38  import javax.servlet.http.HttpServletRequest;
39  import javax.xml.datatype.DatatypeConfigurationException;
40  import javax.xml.datatype.DatatypeFactory;
41  import java.util.Calendar;
42  import javax.xml.ws.RequestWrapper;
43  import javax.xml.ws.ResponseWrapper;
44  import javax.xml.ws.WebServiceContext;
45  import javax.xml.ws.handler.MessageContext;
46  import javax.xml.ws.handler.soap.SOAPMessageContext;
47  import org.miloss.fgsms.common.AuditLogger;
48  import org.miloss.fgsms.common.IpAddressUtility;
49  import org.miloss.fgsms.common.Constants;
50  import org.miloss.fgsms.common.UserIdentityUtil;
51  import org.miloss.fgsms.common.Utility;
52  import org.miloss.fgsms.services.interfaces.common.GetOperatingStatusRequestMessage;
53  import org.miloss.fgsms.services.interfaces.common.GetOperatingStatusResponseMessage;
54  import org.miloss.fgsms.services.interfaces.common.PolicyType;
55  import org.miloss.fgsms.services.interfaces.common.SecurityWrapper;
56  import org.miloss.fgsms.services.interfaces.datacollector.*;
57  import org.miloss.fgsms.services.interfaces.faults.ServiceUnavailableFaultCodes;
58  import org.miloss.fgsms.services.interfaces.policyconfiguration.MachinePolicy;
59  import org.miloss.fgsms.services.interfaces.policyconfiguration.ProcessPolicy;
60  import org.miloss.fgsms.sla.AuxHelper;
61  import org.miloss.fgsms.sla.SLACommon;
62  import org.miloss.fgsms.sla.TransactionalSLAProcessor;
63  import org.apache.log4j.Level;
64  import org.miloss.fgsms.common.DBSettingsLoader;
65  import org.miloss.fgsms.common.DBUtils;
66  import org.miloss.fgsms.common.Logger;
67  import org.miloss.fgsms.services.interfaces.policyconfiguration.KeyNameValueEnc;
68  import us.gov.ic.ism.v2.ClassificationType;
69  
70  /**
71   * fgsms's Data Collector Service, provides a centralized and unified interfaced
72   * for collecting supported data from fgsms's agents
73   *
74   * @author AO
75   */
76  //@javax.ejb.Stateless
77  @WebService(serviceName = "DataCollectorService", name = "DCS",
78          targetNamespace = "urn:org:miloss:fgsms:services:interfaces:dataCollector"
79  //,wsdlLocation = "WEB-INF/wsdl/DCS8.wsdl"
80  )
81  @SOAPBinding(parameterStyle = SOAPBinding.ParameterStyle.WRAPPED, use = SOAPBinding.Use.LITERAL)
82  public class DCS4jBean implements DCS, org.miloss.fgsms.services.interfaces.datacollector.OpStatusService {
83  
84      private static final int MAXTEXT = 128;
85      private static final Logger log = Logger.getLogger("fgsms.DataCollectorService");
86      private static boolean IsRegistered = false;
87      private static Calendar started = null;
88      private static DatatypeFactory df = null;
89      private static long successTX = 0;
90      private static long failuresTX = 0;
91      private static HashMap uriaction = new HashMap();
92      private static HashMap urialturi = new HashMap();
93      private static HashMap hosts = new HashMap();
94      private static HashMap agents = new HashMap();
95      private static boolean DEBUG = false;
96  
97      public DCS4jBean() throws DatatypeConfigurationException {
98          Init();
99      }
100     private static TransactionalSLAProcessor p = new TransactionalSLAProcessor();
101 
102     /**
103      * constructor used for unit testing, do not remove
104      *
105      * @param c2
106      */
107     protected DCS4jBean(WebServiceContext c2) throws DatatypeConfigurationException {
108         ctx = c2;
109         Init();
110     }
111 
112     private synchronized void Init() throws DatatypeConfigurationException {
113         df = DatatypeFactory.newInstance();
114         if (started == null) {
115             GregorianCalendar gcal = new GregorianCalendar();
116             gcal.setTimeInMillis(System.currentTimeMillis());
117             started = (gcal);
118         }
119     }
120     @Resource
121     private WebServiceContext ctx;
122 
123     /**
124      *
125      * @param req
126      * @return returns
127      * org.miloss.fgsms.services.interfaces.datacollector.AddDataResponseMsg
128      * @throws AccessDeniedException
129      * @throws ServiceUnavailableException
130      */
131     @WebMethod(operationName = "AddData", action = "urn:org:miloss:fgsms:services:interfaces:dataCollector/dataCollectorService/AddData")
132     @WebResult(name = "AddDataResult", targetNamespace = "urn:org:miloss:fgsms:services:interfaces:dataCollector")
133     @RequestWrapper(localName = "AddData", targetNamespace = "urn:org:miloss:fgsms:services:interfaces:dataCollector", className = "org.miloss.fgsms.services.interfaces.datacollector.AddData")
134     @ResponseWrapper(localName = "AddDataResponse", targetNamespace = "urn:org:miloss:fgsms:services:interfaces:dataCollector", className = "org.miloss.fgsms.services.interfaces.datacollector.AddDataResponse")
135     public AddDataResponseMsg addData(
136             @WebParam(name = "req", targetNamespace = "urn:org:miloss:fgsms:services:interfaces:dataCollector") AddDataRequestMsg req)
137             throws AccessDeniedException, ServiceUnavailableException {
138         CheckRegistration(ctx.getMessageContext());
139         if (req == null || Utility.stringIsNullOrEmpty(req.getURI())) {
140             failuresTX++;
141             throw new IllegalArgumentException("request or uri is null");
142         }
143         Utility.validateClassification(req.getClassification());
144 
145         String currentUser = UserIdentityUtil.getFirstIdentityToString(ctx);
146         AuditLogger.logItem(this.getClass().getCanonicalName(), "addData", currentUser, req.getURI() + " agent type " + req.getAgentType() + " " + req.getMessage(), req.getClassification(), ctx.getMessageContext());
147         UserIdentityUtil.assertAgentRole(currentUser, "addData", req.getClassification(), ctx);
148         AddDataResponseMsg res = new AddDataResponseMsg();
149         String tid = "";
150         boolean success = true;
151         if (Utility.stringIsNullOrEmpty(req.getTransactionID())) {
152             tid = java.util.UUID.randomUUID().toString();
153         } else {
154             tid = req.getTransactionID();
155         }
156         Connection con = Utility.getPerformanceDBConnection();;
157 
158         if (con == null) {
159             org.miloss.fgsms.services.interfaces.faults.ServiceUnavailableException t = new org.miloss.fgsms.services.interfaces.faults.ServiceUnavailableException();
160             t.setCode(ServiceUnavailableFaultCodes.DATA_BASE_UNAVAILABLE);
161             ServiceUnavailableException e = new ServiceUnavailableException("db unavailable", t);
162             failuresTX++;
163             throw e;
164         }
165         if (success) {
166 
167             if (DEBUG) {
168                 log.log(Level.WARN, "Recording tid " + tid);
169             }
170             success = pushNewDatatoSQL(req, tid, con);
171         }
172 
173         DBUtils.safeClose(con);
174 
175         if (success) {
176             successTX++;
177 
178             updateServiceHostStats(req.getServiceHost(), 1);
179             res.setStatus(DataResponseStatus.SUCCESS);
180             try {
181                 p.ProcessNewTransaction(req, tid);
182             } catch (Exception ex) {
183                 log.log(Level.ERROR, "error caught processing SLAs", ex);
184             }
185         } else {
186             failuresTX++;
187             res.setStatus(DataResponseStatus.FAILURE);
188         }
189         res.setClassification(getCurrentClassificationLevel());
190         return res;
191     }
192 
193     protected static SecurityWrapper getCurrentClassificationLevel() throws ServiceUnavailableException {
194         Connection con = Utility.getConfigurationDBConnection();
195         ResultSet results = null;
196         PreparedStatement comm = null;
197         SecurityWrapper ret = new SecurityWrapper();
198         try {
199 
200             comm = con.prepareStatement("Select classification, caveat from GlobalPolicies;");
201 
202             /////////////////////////////////////////////
203             //get the global policy for data retension
204             /////////////////////////////////////////////
205             results = comm.executeQuery();
206 
207             if (results.next()) {
208                 ret.setClassification(ClassificationType.fromValue(results.getString("classification")));
209                 ret.setCaveats(results.getString("caveat"));
210 
211                 return ret;
212             }
213         } catch (Exception ex) {
214             log.log(Level.ERROR, "Error setting global policy (no records currently exist", ex);
215 
216         } finally {
217             DBUtils.safeClose(results);
218             DBUtils.safeClose(comm);
219             DBUtils.safeClose(con);
220         }
221 
222         try {
223             con = Utility.getConfigurationDBConnection();
224             comm = con.prepareStatement("INSERT INTO GlobalPolicies (PolicyRefreshRate, RecordedMessageCap, classification, caveat) " + " VALUES (?, ?, ?, ?);");
225             comm.setLong(1, 3 * 60 * 100);
226             comm.setLong(2, 1024000);
227             comm.setString(3, "U");
228             comm.setString(4, "None");
229             comm.execute();
230             return ret;
231 
232         } catch (Exception ex) {
233             log.log(Level.ERROR, "Error setting global policy (no records currently exist", ex);
234 
235         } finally {
236             DBUtils.safeClose(comm);
237             DBUtils.safeClose(con);
238         }
239         org.miloss.fgsms.services.interfaces.faults.ServiceUnavailableException t = new org.miloss.fgsms.services.interfaces.faults.ServiceUnavailableException();
240         t.setCode(ServiceUnavailableFaultCodes.DATA_BASE_UNAVAILABLE);
241         ServiceUnavailableException e = new ServiceUnavailableException("db unavailable", t);
242         throw e;
243     }
244 
245     private boolean pushNewDatatoSQL(AddDataRequestMsg req, String tid, Connection con) {
246         if (req == null) {
247             return false;//throw new Exception("Add Request is null");
248         }
249         long now = System.currentTimeMillis();
250         //Driver d;
251         PreparedStatement com = null;
252         try {
253             com = con.prepareStatement("INSERT INTO RawData(ConsumerIdentity, "
254                     + "HostingSource, responseSize, requestSize, URI, Success, MonitorSource, requestxml, "
255                     + "responsexml, UTCdatetime, ResponseTimeMS, originalurl, agenttype, soapaction, TransactionID"
256                     + ", relatedtransactionid, threadid, requestheaders, responseheaders, message" //20 params
257                     + ") VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?);");
258             //int MAXTEXT = 128;
259             if (req.getIdentity() != null && req.getIdentity().size() > 0) {
260                 StringBuilder sb = new StringBuilder();
261 
262                 for (int i = 0; i < req.getIdentity().size(); i++) {
263                     if (!Utility.stringIsNullOrEmpty(req.getIdentity().get(i))) {
264                         sb = sb.append(req.getIdentity().get(i)).append(";");
265                     }
266                 }
267                 //workaround for a bug discovered post RC6 from the ASP.NET Agent
268                 String ids = sb.toString();
269                 if (!Utility.stringIsNullOrEmpty(ids)) {
270                     ids = ids.trim();
271                 }
272                 if (!Utility.stringIsNullOrEmpty(ids)) {
273                     ids = ids.substring(0, ids.length() - 1);
274                     com.setString(1, Utility.truncate((ids), MAXTEXT));
275                 } else {
276                     com.setString(1, "");
277                 }
278             } else {
279                 com.setString(1, "");
280             }
281             com.setString(2, Utility.truncate(req.getServiceHost(), MAXTEXT));
282             com.setLong(3, req.getResponseSize());
283             com.setLong(4, req.getRequestSize());
284             com.setString(5, Utility.truncate(req.getURI(), MAXTEXT));
285             com.setBoolean(6, req.isSuccess());
286             com.setString(7, Utility.truncate(InetAddress.getLocalHost().getHostName().toLowerCase(), MAXTEXT));
287             if (Utility.stringIsNullOrEmpty(req.getXmlRequest())) {
288                 com.setNull(8, java.sql.Types.BINARY);
289             } else {
290                 byte[] x = Utility.EN((req.getXmlRequest()).trim()).getBytes(Constants.CHARSET);// Encoding.UTF8.GetBytes(req.getXmlRequest());
291                 ByteArrayInputStream s = new ByteArrayInputStream(x);
292                 com.setBinaryStream(8, s, (int) x.length);
293             }
294             if (Utility.stringIsNullOrEmpty(req.getXmlResponse())) {
295                 com.setNull(9, java.sql.Types.BINARY);
296             } else {
297                 byte[] x = Utility.EN((req.getXmlResponse()).trim()).getBytes(Constants.CHARSET);// Encoding.UTF8.GetBytes(req.getXmlRequest());
298                 ByteArrayInputStream s = new ByteArrayInputStream(x);
299                 com.setBinaryStream(9, s, (int) x.length);
300             }
301 
302             if (req.getRecordedat() == null) {
303                 com.setLong(10, System.currentTimeMillis());
304             } else {
305                 com.setLong(10, req.getRecordedat().getTimeInMillis());
306             }
307             com.setLong(11, req.getResponseTime());
308             com.setString(12, Utility.truncate(req.getRequestURI(), MAXTEXT));
309             if (Utility.stringIsNullOrEmpty(req.getAgentType())) {
310                 com.setNull(13, java.sql.Types.VARCHAR);
311             } else {
312                 com.setString(13, Utility.truncate(req.getAgentType(), MAXTEXT));
313             }
314 
315             if (Utility.stringIsNullOrEmpty(req.getAction())) {
316                 com.setString(14, "unspecified");
317                 req.setAction("unspecified");
318                 //com.setNull(14, java.sql.Types.VARCHAR);
319             } else {
320                 com.setString(14, Utility.truncate(req.getAction(), MAXTEXT));
321             }
322 
323             com.setString(15, tid);
324 
325             if (Utility.stringIsNullOrEmpty(req.getRelatedTransactionID())) {
326                 com.setNull(16, java.sql.Types.VARCHAR);
327             } else {
328                 com.setString(16, Utility.truncate(req.getRelatedTransactionID(), MAXTEXT));
329             }
330 
331             if (Utility.stringIsNullOrEmpty(req.getTransactionThreadID())) {
332                 com.setNull(17, java.sql.Types.VARCHAR);
333             } else {
334                 com.setString(17, Utility.truncate(req.getTransactionThreadID(), MAXTEXT));
335             }
336             if (req.getHeadersRequest() == null || req.getHeadersRequest().isEmpty()) {
337                 com.setNull(18, java.sql.Types.BINARY);
338             } else {
339                 StringBuilder data = new StringBuilder();
340 
341                 for (int i = 0; i < req.getHeadersRequest().size(); i++) {
342                     if (req.getHeadersRequest().get(i) != null) {
343                         if (!Utility.stringIsNullOrEmpty(req.getHeadersRequest().get(i).getName())) {
344                             if (req.getHeadersRequest().get(i).getName().equalsIgnoreCase("Authorization")) {
345                                 data.append(req.getHeadersRequest().get(i).getName()).append("=***************");
346                             } else {
347                                 data.append(req.getHeadersRequest().get(i).getName()).append("=");
348                                 for (int k = 0; k < req.getHeadersRequest().get(i).getValue().size(); k++) {
349                                     data.append(req.getHeadersRequest().get(i).getValue().get(k)).append(";");
350                                 }
351                             }
352                             data.append("|");
353                         }
354                     }
355                 }
356                 //  if (data.length() > 0) {
357                 //      data = data.substring(0, data.length() - 2);
358                 //   }
359                 String d2 = Utility.EN(data.toString());
360                 com.setObject(18, (d2).getBytes(Constants.CHARSET));
361             }
362             if (req.getHeadersResponse() == null || req.getHeadersResponse().isEmpty()) {
363                 com.setNull(19, java.sql.Types.BINARY);
364             } else {
365                 StringBuilder data = new StringBuilder();
366                 for (int i = 0; i < req.getHeadersResponse().size(); i++) {
367                     if (req.getHeadersResponse().get(i) != null) {
368                         if (!Utility.stringIsNullOrEmpty(req.getHeadersResponse().get(i).getName())) {
369                             //password scrubber
370                             if (req.getHeadersResponse().get(i).getName().equalsIgnoreCase("Authorization")) {
371                                 data.append(req.getHeadersResponse().get(i).getName()).append("=");
372                                 data.append("***************");
373                             } else {
374                                 data.append(req.getHeadersResponse().get(i).getName()).append("=");
375                                 for (int k = 0; k < req.getHeadersResponse().get(i).getValue().size(); k++) {
376                                     data.append(req.getHeadersResponse().get(i).getValue().get(k)).append(";");
377                                 }
378                             }
379                             data.append("|");
380                         }
381                     }
382                 }
383                 // if (data.length() > 0) {
384                 //    data = data.substring(0, data.length() - 2);
385                 // }
386                 String d2 = Utility.EN(data.toString());
387                 com.setObject(19, (d2).getBytes(Constants.CHARSET));
388             }
389 
390             if (req.getMessage() == null || req.getMessage().isEmpty()) {
391                 com.setNull(20, java.sql.Types.BINARY);
392             } else {
393 
394                 com.setObject(20, (req.getMessage()).getBytes(Constants.CHARSET));
395             }
396             try {
397                 com.execute();
398             } catch (Exception ex) {
399                 log.log(Level.ERROR, "DCS error saving transactional data " + ex.getMessage());
400                 log.log(Level.DEBUG, "DCS error saving transactional data ", ex);
401             } finally {
402                 DBUtils.safeClose(com);
403             }
404             if (!urialturi.containsKey(req.getURI() + req.getRequestURI())) {
405                 urialturi.put(req.getURI() + req.getRequestURI(), true);
406                 try {
407                     com = con.prepareStatement("insert into alternateurls (uri, alturi) values (?,?);");
408                     com.setString(1, req.getURI());
409                     com.setString(2, req.getRequestURI());
410                     com.execute();
411                     log.log(Level.DEBUG, "DCS DEBUG adding alt URL");
412                 } catch (Exception ex) {
413                     log.log(Level.DEBUG, "DCS DEBUG adding alt URL FAILED");
414                 } finally {
415                     DBUtils.safeClose(com);
416                 }
417             }
418             if (!uriaction.containsKey(req.getURI() + req.getAction())) {
419                 uriaction.put(req.getURI() + req.getAction(), true);
420                 try {
421                     com = con.prepareStatement("insert into actionlist (uri, soapaction) values (?,?);");
422                     com.setString(1, req.getURI());
423                     com.setString(2, req.getAction());
424                     com.execute();
425                     log.log(Level.DEBUG, "DCS DEBUG adding URL action");
426                 } catch (Exception ex) {
427                     log.log(Level.DEBUG, "DCS DEBUG adding URL action FAILED");
428                 } finally {
429                     DBUtils.safeClose(com);
430                 }
431             }
432 
433             if (!agents.containsKey(req.getAgentType())) {
434                 agents.put(req.getAgentType(), true);
435                 Connection config = Utility.getConfigurationDBConnection();
436                 try {
437                     com = config.prepareStatement("INSERT INTO agents(agenttype)  VALUES (?)");
438                     com.setString(1, req.getAgentType());
439                     com.execute();
440                     log.log(Level.DEBUG, "DCS DEBUG adding agent type");
441                 } catch (Exception ex) {
442                     log.log(Level.DEBUG, "DCS DEBUG adding agent type FAILED");
443                 } finally {
444                     DBUtils.safeClose(com);
445                     DBUtils.safeClose(config);
446                 }
447             }
448 
449             now = System.currentTimeMillis() - now;
450             log.log(Level.DEBUG, "SQL Data Push for " + req.getURI() + " in " + now + "ms");
451 
452         } catch (Exception ex) {
453             String msg = ex.getMessage();
454             if (msg.contains("rawdata_pkey")) {
455                 //this means that there was an issue in agent land, basically, the agent is continuously trying to transmit the same data to this DCS instance over and over again. Please
456                 //report this to the developers
457                 log.log(Level.WARN, "Error caught recording performance data. "
458                         + "this means that there was an issue in agent land, basically, the agent is continuously trying to transmit the same data to this DCS instance over and over again. Please report this to the developers URL: "
459                         + req.getURI() + " Action: " + req.getAction() + " Agent: " + req.getAgentType() + " Host: " + req.getServiceHost() + " I'm going to return this as successful, even though it was not.");
460                 return true;
461             } else {
462                 log.log(Level.ERROR, "Error recording performance data. Data may be lost URL: " + req.getURI() + " Action: " + req.getAction() + " Agent: " + req.getAgentType() + " Host: " + req.getServiceHost(), ex);
463                 return false;
464             }
465         } finally {
466           
467 
468         }
469 
470         return true;
471     }
472 
473     /**
474      *
475      * @param req
476      * @return returns
477      * org.miloss.fgsms.services.interfaces.datacollector.AddDataResponseMsg
478      * @throws AccessDeniedException
479      * @throws ServiceUnavailableException
480      */
481     @WebMethod(operationName = "AddMoreData", action = "urn:org:miloss:fgsms:services:interfaces:dataCollector/dataCollectorService/AddMoreData")
482     @WebResult(name = "AddDataResult", targetNamespace = "urn:org:miloss:fgsms:services:interfaces:dataCollector")
483     @RequestWrapper(localName = "AddMoreData", targetNamespace = "urn:org:miloss:fgsms:services:interfaces:dataCollector", className = "org.miloss.fgsms.services.interfaces.datacollector.AddMoreData")
484     @ResponseWrapper(localName = "AddMoreDataResponse", targetNamespace = "urn:org:miloss:fgsms:services:interfaces:dataCollector", className = "org.miloss.fgsms.services.interfaces.datacollector.AddMoreDataResponse")
485     public AddDataResponseMsg addMoreData(
486             @WebParam(name = "req", targetNamespace = "urn:org:miloss:fgsms:services:interfaces:dataCollector") List<AddDataRequestMsg> req)
487             throws AccessDeniedException, ServiceUnavailableException {
488         CheckRegistration(ctx.getMessageContext());
489         String currentUser = UserIdentityUtil.getFirstIdentityToString(ctx);
490         if (req == null || req.isEmpty())// Utility.stringIsNullOrEmpty(req.getURI())) {
491         {
492             AuditLogger.logItem(this.getClass().getCanonicalName(), "addMoreData", currentUser, "null or empty request ", "not specified", ctx.getMessageContext());
493             throw new IllegalArgumentException("request is empty");
494         }
495         if (req.get(0) == null || req.get(0).getClassification() == null || req.get(0).getClassification().getClassification() == null) {
496             throw new IllegalArgumentException("a classificaiton level must be specified for at least the first item");
497 
498         }
499 
500         AuditLogger.logItem(this.getClass().getCanonicalName(), "addMoreData", currentUser, "msg count=" + req.size() + " 1st record agent type=" + req.get(0).getAgentType() + " uri=" + req.get(0).getURI(), req.get(0).getClassification(), ctx.getMessageContext());
501         UserIdentityUtil.assertAgentRole(currentUser, "addData", req.get(0).getClassification(), ctx);
502         Connection con = null;
503         boolean success = true;
504 
505         con = Utility.getPerformanceDBConnection();
506 
507         if (con == null) {
508             org.miloss.fgsms.services.interfaces.faults.ServiceUnavailableException t = new org.miloss.fgsms.services.interfaces.faults.ServiceUnavailableException();
509             t.setCode(ServiceUnavailableFaultCodes.DATA_BASE_UNAVAILABLE);
510             ServiceUnavailableException e = new ServiceUnavailableException("db unavailable", t);
511             throw e;
512         }
513         AddDataResponseMsg res = new AddDataResponseMsg();
514         String tid = "";
515         for (int i = 0; i < req.size(); i++) {
516             if (Utility.stringIsNullOrEmpty(req.get(i).getTransactionID())) {
517                 tid = java.util.UUID.randomUUID().toString();
518             } else {
519                 tid = req.get(i).getTransactionID();
520             }
521             if (DEBUG) {
522                 log.log(Level.WARN, "Recording tid " + tid);
523             }
524             boolean t = pushNewDatatoSQL(req.get(i), tid, con);
525             success = success && t;
526             if (t) {
527                 try {
528                     p.ProcessNewTransaction(req.get(i), tid);
529                 } catch (Exception ex) {
530                     log.log(Level.ERROR, "error caught processing SLAs", ex);
531                 }
532                 updateServiceHostStats(req.get(0).getServiceHost(), req.size());
533             }
534         }
535         DBUtils.safeClose(con);
536 
537         if (success) {
538             successTX += req.size();
539             res.setStatus(DataResponseStatus.SUCCESS);
540         } else {
541             failuresTX += req.size();
542             res.setStatus(DataResponseStatus.FAILURE);
543         }
544         try {
545             res.setClassification(getCurrentClassificationLevel());
546         } catch (Exception ex) {
547             log.log(Level.WARN, null, ex);
548         }
549         return res;
550     }
551 
552     private void updateServiceHostStats(String host, int count) {
553         Connection config = Utility.getConfigurationDBConnection();
554         PreparedStatement com = null;
555         if (!hosts.containsKey(host)) {
556             hosts.put(host, true);
557             try {
558                 com = config.prepareStatement("INSERT INTO servicehosts(hostname)  VALUES (?)");
559                 com.setString(1, host);
560                 com.execute();
561 
562                 if (DEBUG) {
563                     log.log(Level.DEBUG, "DCS DEBUG adding service host");
564                 }
565             } catch (Exception ex) {
566                 log.log(Level.DEBUG, "DCS DEBUG adding service host FAILED " + ex.getMessage(), ex);
567             } finally {
568                 DBUtils.safeClose(com);
569             }
570         }
571         try {
572             com = config.prepareStatement("UPDATE servicehosts   SET records=records + ? WHERE hostname=?; "
573                     + "UPDATE dcsservicehosts   SET  records=records + ? WHERE hostname=? ;  ");
574             com.setInt(1, count);
575             com.setString(2, host);
576             com.setInt(3, count);
577             com.setString(4, Utility.getHostName());
578             com.execute();
579 
580             if (DEBUG) {
581                 log.log(Level.INFO, "DCS DEBUG updated servicehost stats");
582             }
583         } catch (Exception ex) {
584             log.log(Level.WARN, "Trouble updating statistics for either servicehsots or dscservicehosts", ex);
585         } finally {
586             DBUtils.safeClose(com);
587             DBUtils.safeClose(config);
588         }
589 
590     }
591 
592     /**
593      * checks the registration of this instance of this service by recreating a
594      * service policy based on the modified url of an inbound request
595      *
596      * @param messageContext
597      */
598     private void CheckRegistration(MessageContext messageContext) {
599         if (!IsRegistered) {
600             if (messageContext != null) {
601                 HttpServletRequest ctxg = null;
602                 try {
603                     ctxg = (HttpServletRequest) messageContext.get(SOAPMessageContext.SERVLET_REQUEST);
604                 } catch (Exception ex) {
605                 }
606                 String requestURL = "";
607                 //     if (Utility.stringIsNullOrEmpty(requestURL)) {
608                 try {
609                     Object j = messageContext.get("javax.xml.ws.service.endpoint.address");
610                     if (j != null) {
611                         requestURL = (String) j;
612                     }
613                 } catch (Exception ex) {
614                     // log.log(Level.WARN, "fgsms error getting request url " + ex.getLocalizedMessage());
615                 }
616                 //        }
617                 if ((Utility.stringIsNullOrEmpty(requestURL)) && ctxg != null) {
618                     try {
619                         StringBuffer buff = ctxg.getRequestURL();
620                         requestURL = buff.toString();
621                     } catch (Exception ex) {
622                         //log.log(Level.WARN, "fgsms error getting request url " + ex.getLocalizedMessage());
623                     }
624                 }
625                 if (Utility.stringIsNullOrEmpty(requestURL)) {
626                     log.log(Level.WARN, "Unable to self register this fgsms DCS service, giving up");
627                     return;
628                 }
629                 requestURL = IpAddressUtility.modifyURL(requestURL, false);
630                 Connection con = Utility.getConfigurationDBConnection();
631                 try {
632 
633                     AuxHelper.CheckPolicyAndCreate(requestURL, con, PolicyType.TRANSACTIONAL, true, AuxHelper.UNSPECIFIED, Utility.getHostName());
634 
635                     log.log(Level.INFO, "Successful self registration of this DCS at " + requestURL);
636                 } catch (Exception ex) {
637                     log.log(Level.WARN, "Unable to self register this DCS service at " + requestURL, ex);
638                 } finally {
639                     DBUtils.safeClose(con);
640                 }
641                 IsRegistered = true;
642             } else {
643                 log.log(Level.INFO, "unable to self register this DCS, the context object is null.");
644             }
645 
646             Connection con = Utility.getConfigurationDBConnection();
647             PreparedStatement cmd = null;
648             try {
649                 cmd = con.prepareStatement("INSERT INTO dcsservicehosts(hostname)    VALUES (?);");
650                 cmd.setString(1, Utility.getHostName());
651                 cmd.executeUpdate();
652                 cmd.close();
653             } catch (Exception ex) {
654                 log.log(Level.DEBUG, "error caught inserting dcs record into dcsservicehosts", ex);
655             } finally {
656                 DBUtils.safeClose(cmd);
657                 DBUtils.safeClose(con);
658             }
659 
660         }
661     }
662 
663     /**
664      *
665      * @param addStatisticalDataRequestMsg
666      * @return returns
667      * org.miloss.fgsms.services.interfaces.datacollector.AddStatisticalDataResponseMsg
668      * @throws AccessDeniedException
669      * @throws ServiceUnavailableException
670      */
671     @WebMethod(operationName = "AddStatisticalData", action = "urn:org:miloss:fgsms:services:interfaces:dataCollector/dataCollectorService/AddStatisticalData")
672     @WebResult(name = "AddStatisticalDataResponseMsg", targetNamespace = "urn:org:miloss:fgsms:services:interfaces:dataCollector")
673     @RequestWrapper(localName = "AddStatisticalData", targetNamespace = "urn:org:miloss:fgsms:services:interfaces:dataCollector", className = "org.miloss.fgsms.services.interfaces.datacollector.AddStatisticalData")
674     @ResponseWrapper(localName = "AddStatisticalDataResponse", targetNamespace = "urn:org:miloss:fgsms:services:interfaces:dataCollector", className = "org.miloss.fgsms.services.interfaces.datacollector.AddStatisticalDataResponse")
675     @SuppressWarnings("unchecked")
676     public AddStatisticalDataResponseMsg addStatisticalData(
677             @WebParam(name = "AddStatisticalDataRequestMsg", targetNamespace = "urn:org:miloss:fgsms:services:interfaces:dataCollector") AddStatisticalDataRequestMsg req)
678             throws AccessDeniedException, ServiceUnavailableException {
679 
680         CheckRegistration(ctx.getMessageContext());
681 
682         String currentUser = UserIdentityUtil.getFirstIdentityToString(ctx);
683         if (req == null)// Utility.stringIsNullOrEmpty(req.getURI())) {
684         {
685             AuditLogger.logItem(this.getClass().getCanonicalName(), "addStatisticalData", currentUser, "null or empty request ", "not specified", ctx.getMessageContext());
686             throw new IllegalArgumentException("request is empty");
687         }
688         Utility.validateClassification(req.getClassification());
689 
690         if (Utility.stringIsNullOrEmpty(req.getAgentType())) {
691             throw new IllegalArgumentException("A agent type must be specified");
692         }
693         if (Utility.stringIsNullOrEmpty(req.getBrokerURI())) {
694             throw new IllegalArgumentException("A uri must be specified");
695         }
696         if (Utility.stringIsNullOrEmpty(req.getBrokerHostname())) {
697             throw new IllegalArgumentException("A hostname must be specified");
698         }
699 
700         AuditLogger.logItem(this.getClass().getCanonicalName(), "addStatisticalData", currentUser, "uri=" + req.getBrokerURI() + " " + req.getAgentType(), req.getClassification(), ctx.getMessageContext());
701         UserIdentityUtil.assertAgentRole(currentUser, "addStatisticalData", req.getClassification(), ctx);
702 
703         Connection config = Utility.getConfigurationDBConnection();
704         PreparedStatement com = null;
705         try {
706             if (!agents.containsKey(req.getAgentType())) {
707                 agents.put(req.getAgentType(), true);
708                 try {
709                     com = config.prepareStatement("INSERT INTO agents(agenttype)  VALUES (?)");
710                     com.setString(1, req.getAgentType());
711                     com.execute();
712                     com.close();
713                     log.log(Level.DEBUG, "DCS DEBUG adding agent type");
714                 } catch (Exception ex) {
715                     log.log(Level.DEBUG, "DCS DEBUG adding agent type FAILED");
716                 }
717             }
718             AuxHelper.CheckStatisticalPolicyAndCreate(req.getBrokerURI(), config, true, AuxHelper.UNSPECIFIED, req.getBrokerHostname());
719         } catch (Exception ex) {
720             log.log(Level.DEBUG, null, ex);
721         } finally {
722             DBUtils.safeClose(com);
723             DBUtils.safeClose(config);
724         }
725 
726         Connection con = Utility.getPerformanceDBConnection();
727         boolean error = false;
728         if (con == null) {
729             org.miloss.fgsms.services.interfaces.faults.ServiceUnavailableException t = new org.miloss.fgsms.services.interfaces.faults.ServiceUnavailableException();
730             t.setCode(ServiceUnavailableFaultCodes.DATA_BASE_UNAVAILABLE);
731             ServiceUnavailableException e = new ServiceUnavailableException("db unavailable", t);
732             throw e;
733         }
734         try {
735 
736             AuxHelper.TryUpdateStatus(req.isOperationalStatus(), req.getBrokerURI(), req.getOperationalStatusMessage(), true, PolicyType.STATISTICAL, "unspecified", req.getBrokerHostname());
737 
738             //this is necessary for when a queue/topic is no longer active
739             com = con.prepareStatement("delete from brokerrawdata where host=?");
740             com.setString(1, req.getBrokerURI());
741             com.execute();
742             for (int i = 0; i < req.getData().size(); i++) {
743                 updateBrokerData(req.getBrokerURI(), req.getData().get(i).getQueueOrTopicName(), req.getData().get(i).getQueueOrTopicCanonicalName(),
744                         req.getData().get(i).getMessagesOut(), req.getData().get(i).getMessagesIn(), req.getData().get(i).getTotalConsumers(), req.getData().get(i).getActiveConsumers(),
745                         req.getData().get(i).getDepth(), req.getData().get(i).getItemType(), req.getData().get(i).getBytesIn(), req.getData().get(i).getBytesOut(), req.getData().get(i).getBytesDropped(),
746                         req.getData().get(i).getMessagesDropped(), con, req.getAgentType());
747             }
748             //Do standard SLA rules
749             SLACommon.ProcessStatisticalSLARules(req, true);
750 
751         } catch (Exception ex) {
752             log.log(Level.WARN, "error saving broker data", ex);
753             error = true;
754         } finally {
755             DBUtils.safeClose(com);
756             DBUtils.safeClose(con);
757         }
758 
759         if (error) {
760             failuresTX++;
761             org.miloss.fgsms.services.interfaces.faults.ServiceUnavailableException t = new org.miloss.fgsms.services.interfaces.faults.ServiceUnavailableException();
762             t.setCode(ServiceUnavailableFaultCodes.UNEXPECTED_ERROR);
763             ServiceUnavailableException e = new ServiceUnavailableException("", t);
764             throw e;
765         }
766         successTX += req.getData().size();
767         AddStatisticalDataResponseMsg res = new AddStatisticalDataResponseMsg();
768         res.setClassification(getCurrentClassificationLevel());
769         res.setSuccess(true);
770         return res;
771     }
772 
773     private void updateBrokerData(String url, String name, String bigname,
774             long MessageCount, long RecievedMessageCount, long ConsumerCount, long ActiveConsumerCount,
775             long QueueDepth, String ExchangeType, long bytesin, long bytesout, long bytesdrop,
776             long MessageDropCount, Connection perf, String agenttype) {
777 
778         if (!Utility.stringIsNullOrEmpty(name)) {
779             PreparedStatement comm = null;
780             try {
781                 comm = perf.prepareStatement("Delete from brokerrawdata where host=? and namecol=? and canonicalname=?; "
782                         + "INSERT INTO brokerrawdata (host, utcdatetime, namecol, canonicalname, messagecount,recievedmessagecount, consumercount, "
783                         + "activeconsumercount, queuedepth, typecol, agenttype, messagedropcount, bytesdropcount, bytesin, bytesout) "
784                         + "VALUES (?, ?, ?,?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);"
785                         + "INSERT INTO brokerhistory (host, utcdatetime, namecol,canonicalname, messagecount,recievedmessagecount, consumercount, "
786                         + "activeconsumercount, queuedepth, typecol, agenttype, messagedropcount, bytesdropcount, bytesin, bytesout) "
787                         + "VALUES (?, ?, ?,?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?);");
788                 comm.setString(1, Utility.truncate(url, MAXTEXT));
789                 comm.setString(2, Utility.truncate(name, MAXTEXT));
790                 comm.setString(3, Utility.truncate(bigname, MAXTEXT));
791                 comm.setString(4, Utility.truncate(url, MAXTEXT));
792                 comm.setLong(5, System.currentTimeMillis());
793                 comm.setString(6, Utility.truncate(name, MAXTEXT));
794                 comm.setString(7, Utility.truncate(bigname, MAXTEXT));
795                 comm.setLong(8, MessageCount);
796                 comm.setLong(9, RecievedMessageCount);
797                 comm.setLong(10, ConsumerCount);
798                 comm.setLong(11, ActiveConsumerCount);
799                 comm.setLong(12, QueueDepth);
800                 comm.setString(13, Utility.truncate(ExchangeType, MAXTEXT));
801                 comm.setString(14, Utility.truncate(agenttype, MAXTEXT));
802                 comm.setLong(15, MessageDropCount);
803                 comm.setLong(16, bytesdrop);
804                 comm.setLong(17, bytesin);
805                 comm.setLong(18, bytesout);
806 
807                 comm.setString(19, Utility.truncate(url, MAXTEXT));
808                 comm.setLong(20, System.currentTimeMillis());
809                 comm.setString(21, Utility.truncate(name, MAXTEXT));
810                 comm.setString(22, Utility.truncate(bigname, MAXTEXT));
811                 comm.setLong(23, MessageCount);
812                 comm.setLong(24, RecievedMessageCount);
813                 comm.setLong(25, ConsumerCount);
814                 comm.setLong(26, ActiveConsumerCount);
815                 comm.setLong(27, QueueDepth);
816                 comm.setString(28, Utility.truncate(ExchangeType, MAXTEXT));
817                 comm.setString(29, Utility.truncate(agenttype, MAXTEXT));
818                 comm.setLong(30, MessageDropCount);
819                 comm.setLong(31, bytesdrop);
820                 comm.setLong(32, bytesin);
821                 comm.setLong(33, bytesout);
822                 comm.execute();
823                 comm.close();
824             } catch (SQLException ex) {
825                 log.log(Level.WARN, null, ex);
826             } finally {
827                 DBUtils.safeClose(comm);
828             }
829 
830         }
831 
832     }
833 
834     /**
835      * AddMachineAndProcessData, adds a set of performance records from a set of
836      * monitored processes from a given machine
837      *
838      * @param req
839      * @param addMachineAndProcessDataRequestMsg
840      * @return returns
841      * org.miloss.fgsms.services.interfaces.datacollector.AddMachineAndProcessDataResponseMsg
842      * @throws AccessDeniedException
843      * @throws ServiceUnavailableException
844      */
845     @WebMethod(operationName = "AddMachineAndProcessData", action = "urn:org:miloss:fgsms:services:interfaces:dataCollector/dataCollectorService/AddMachineAndProcessData")
846     @WebResult(name = "AddMachineAndProcessDataResponseMsg", targetNamespace = "urn:org:miloss:fgsms:services:interfaces:dataCollector")
847     @RequestWrapper(localName = "AddMachineAndProcessData", targetNamespace = "urn:org:miloss:fgsms:services:interfaces:dataCollector", className = "org.miloss.fgsms.services.interfaces.datacollector.AddMachineAndProcessData")
848     @ResponseWrapper(localName = "AddMachineAndProcessDataResponse", targetNamespace = "urn:org:miloss:fgsms:services:interfaces:dataCollector", className = "org.miloss.fgsms.services.interfaces.datacollector.AddMachineAndProcessDataResponse")
849     @SuppressWarnings("unchecked")
850     public AddMachineAndProcessDataResponseMsg addMachineAndProcessData(
851             @WebParam(name = "AddMachineAndProcessDataRequestMsg", targetNamespace = "urn:org:miloss:fgsms:services:interfaces:dataCollector") AddMachineAndProcessDataRequestMsg req)
852             throws AccessDeniedException, ServiceUnavailableException {
853 
854         CheckRegistration(ctx.getMessageContext());
855 
856         String currentUser = UserIdentityUtil.getFirstIdentityToString(ctx);
857         if (req == null)// Utility.stringIsNullOrEmpty(req.getURI())) {
858         {
859             AuditLogger.logItem(this.getClass().getCanonicalName(), "addMachineAndProcessData", currentUser, "null or empty request ", "not specified", ctx.getMessageContext());
860             throw new IllegalArgumentException("request is empty");
861         }
862         Utility.validateClassification(req.getClassification());
863         if (Utility.stringIsNullOrEmpty(req.getHostname())) {
864             throw new IllegalArgumentException("A hostname must be specified");
865         }
866         if (Utility.stringIsNullOrEmpty(req.getDomainname())) {
867             throw new IllegalArgumentException("A domain name or 'unspecified' must be specified");
868         }
869 
870         AuditLogger.logItem(this.getClass().getCanonicalName(), "addMachineAndProcessData", currentUser, "uri=" + req.getHostname(), req.getClassification(), ctx.getMessageContext());
871         UserIdentityUtil.assertAgentRole(currentUser, "addMachineAndProcessData", req.getClassification(), ctx);
872 
873         if (req.getMachineData() == null) {
874             throw new IllegalArgumentException("The element MachineData must be specified and contain at least the current status information");
875         }
876         if (Utility.stringIsNullOrEmpty(req.getMachineData().getUri())) {
877             throw new IllegalArgumentException("The element MachineData must specify a uri representing the system, typically in the format urn:hostname:system");
878         }
879 
880         Connection con = null;
881         long now = System.currentTimeMillis();
882         UUID id = UUID.randomUUID();
883         boolean ok = true;
884         PreparedStatement com = null;
885         try {
886             con = Utility.getPerformanceDBConnection();
887 
888             //machine level data
889             com = con.prepareStatement("INSERT INTO rawdatamachineprocess "
890                     + "(uri, memoryused, percentcpu, id, utcdatetime, threads, startedat)    "
891                     + "VALUES (?, ?, ?, ?,     ? ,?, ?);");
892             com.setString(1, Utility.truncate(req.getMachineData().getUri(), MAXTEXT));
893             if (req.getMachineData().getBytesusedMemory() == null) {
894                 com.setNull(2, java.sql.Types.BIGINT);
895             } else {
896                 com.setLong(2, req.getMachineData().getBytesusedMemory());
897             }
898             /*
899              * if (req.getMachineData().getKilobytespersecondNetwork() == null)
900              * { com.setNull(3, java.sql.Types.BIGINT); } else { com.setLong(3,
901              * req.getMachineData().getKilobytespersecondNetwork().longValue());
902              * } if (req.getMachineData().getKilobytespersecondDisk() == null) {
903              * com.setNull(4, java.sql.Types.BIGINT); } else { com.setLong(4,
904              * req.getMachineData().getKilobytespersecondDisk().longValue()); }
905              */
906             if (req.getMachineData().getPercentusedCPU() == null) {
907                 com.setNull(3, java.sql.Types.INTEGER);
908             } else {
909                 com.setLong(3, req.getMachineData().getPercentusedCPU().intValue());
910             }
911 
912             req.getMachineData().setId(id.toString());
913             com.setString(4, id.toString());
914             if (req.getMachineData().getTimestamp() == null) {
915                 com.setLong(5, now);
916             } else {
917                 com.setLong(5, req.getMachineData().getTimestamp().getTimeInMillis());
918             }
919             if (req.getMachineData().getNumberofActiveThreads() == null) {
920                 com.setNull(6, java.sql.Types.BIGINT);
921             } else {
922                 com.setLong(6, req.getMachineData().getNumberofActiveThreads());
923             }
924 
925             if (req.getMachineData().getStartedAt() == null) {
926                 com.setNull(7, java.sql.Types.BIGINT);
927             } else {
928                 com.setLong(7, req.getMachineData().getStartedAt().getTimeInMillis());
929             }
930 
931             com.executeUpdate();
932         } catch (Exception ex) {
933             log.log(Level.ERROR, "DCS unable to save machine data " + ex.getMessage());
934             log.log(Level.DEBUG, "DCS unable to save machine data " + ex.getMessage(), ex);
935         } finally {
936             DBUtils.safeClose(com);
937         }
938 
939         //process level data, if any
940         for (int i = 0; i < req.getProcessData().size(); i++) {
941             try {
942                 com = con.prepareStatement("INSERT INTO rawdatamachineprocess "
943                         + "(uri, memoryused, percentcpu, id, utcdatetime, threads, openfiles, startedat)    "
944                         + "VALUES (?, ?, ?, ?,     ?, ?, ?, ?);");
945                 com.setString(1, Utility.truncate(req.getProcessData().get(i).getUri(), MAXTEXT));
946                 if (req.getProcessData().get(i).getBytesusedMemory() == null) {
947                     com.setNull(2, java.sql.Types.BIGINT);
948                 } else {
949                     com.setLong(2, req.getProcessData().get(i).getBytesusedMemory().longValue());
950                 }
951 
952                 if (req.getProcessData().get(i).getPercentusedCPU() == null) {
953                     com.setNull(3, java.sql.Types.INTEGER);
954                 } else {
955                     com.setLong(3, req.getProcessData().get(i).getPercentusedCPU().intValue());
956                 }
957 
958                 id = UUID.randomUUID();
959                 req.getProcessData().get(i).setId(id.toString());
960                 com.setString(4, id.toString());
961                 if (req.getProcessData().get(i).getTimestamp() == null) {
962                     com.setLong(5, now);
963                 } else {
964                     com.setLong(5, req.getProcessData().get(i).getTimestamp().getTimeInMillis());
965                 }
966                 if (req.getProcessData().get(i).getNumberofActiveThreads() == null) {
967                     com.setNull(6, java.sql.Types.BIGINT);
968                 } else {
969                     com.setLong(6, req.getProcessData().get(i).getNumberofActiveThreads());
970                 }
971                 //  com.setLong(6, req.getProcessData().get(i).getNumberofActiveThreads());
972                 com.setLong(7, req.getProcessData().get(i).getOpenFileHandles());
973 
974                 if (req.getProcessData().get(i).getStartedAt() == null) {
975                     com.setNull(8, java.sql.Types.BIGINT);
976                 } else {
977                     com.setLong(8, now);
978                 }
979                 com.executeUpdate();
980             } catch (Exception ex) {
981                 log.log(Level.ERROR, "DCS unable to save process data " + ex.getMessage());
982                 log.log(Level.DEBUG, "DCS unable to save process data " + ex.getMessage(), ex);
983                 failuresTX++;
984             } finally {
985                 DBUtils.safeClose(com);
986             }
987         }
988 
989         //do network information
990         if (!req.getMachineData().getNetworkAdapterPerformanceData().isEmpty()) {
991             for (int i = 0; i < req.getMachineData().getNetworkAdapterPerformanceData().size(); i++) {
992                 try {
993                     com = con.prepareStatement("INSERT INTO rawdatanic"
994                             + "(nicid, hostname,domainname, utcdatetime, sendkbs, receivekbs, id, uri)    "
995                             + "VALUES (?, ?, ?, ?, ?,?, ?, ?);");
996                     com.setString(1, Utility.truncate(req.getMachineData().getNetworkAdapterPerformanceData().get(i).getAdapterName(), MAXTEXT));
997                     com.setString(2, Utility.truncate(req.getHostname(), MAXTEXT));
998                     com.setString(3, Utility.truncate(req.getDomainname(), MAXTEXT));
999 
1000                     com.setLong(4, now);
1001 
1002                     com.setLong(5, req.getMachineData().getNetworkAdapterPerformanceData().get(i).getKilobytespersecondNetworkTransmit());
1003                     com.setLong(6, req.getMachineData().getNetworkAdapterPerformanceData().get(i).getKilobytespersecondNetworkReceive());
1004 
1005                     com.setString(7, UUID.randomUUID().toString());
1006                     com.setString(8, req.getMachineData().getUri());
1007 
1008                     com.executeUpdate();
1009                 } catch (Exception ex) {
1010                     log.log(Level.ERROR, "DCS unable to save machine/nic data " + ex.getMessage());
1011                     log.log(Level.DEBUG, "DCS unable to save machine/nicdata " + ex.getMessage(), ex);
1012                     failuresTX++;
1013                 } finally {
1014                     DBUtils.safeClose(com);
1015                 }
1016             }
1017         }
1018         //@since 6.3
1019         if (req.getSensorData() != null) {
1020             for (int i = 0; i < req.getSensorData().getItems().size(); i++) {
1021                 try {
1022                     com = con.prepareStatement("INSERT INTO "
1023                             + "rawdatamachinesensor (uri, id, utcdatetime, dataname, datavalue)"
1024                             + "    VALUES (?, ?, ?, ?, ?);");
1025                     com.setString(1, Utility.truncate(req.getMachineData().getUri(), MAXTEXT));
1026                     com.setString(2, UUID.randomUUID().toString());
1027                     com.setLong(3, now);
1028                     com.setString(4, Utility.truncate(req.getSensorData().getItems().get(i).getName(), MAXTEXT));
1029                     com.setString(5, Utility.truncate(req.getSensorData().getItems().get(i).getValue(), MAXTEXT));
1030                     com.executeUpdate();
1031                 } catch (Exception ex) {
1032                     log.log(Level.ERROR, "DCS unable to save machine/sensor data " + ex.getMessage());
1033                     log.log(Level.DEBUG, "DCS unable to save machine/sensor data " + ex.getMessage(), ex);
1034                     failuresTX++;
1035                 } finally {
1036                     DBUtils.safeClose(com);
1037                 }
1038 
1039             }
1040         }
1041 
1042         //do drive information
1043         if (!req.getMachineData().getDriveInformation().isEmpty()) {
1044             for (int i = 0; i < req.getMachineData().getDriveInformation().size(); i++) {
1045                 try {
1046                     com = con.prepareStatement("INSERT INTO "
1047                             + "rawdatadrives(hostname,driveidentifier, freespace, utcdatetime, id,writekbs, readkbs, deviceidentifier, status, statusmsg, domainname, uri)"
1048                             + "    VALUES (?, ?, ?, ?, ?, ?,  ?, ?, ? ,?,?, ?);");
1049                     com.setString(1, Utility.truncate(req.getHostname(), MAXTEXT));
1050                     com.setString(2, Utility.truncate(req.getMachineData().getDriveInformation().get(i).getPartition(), MAXTEXT));
1051                     if (req.getMachineData().getDriveInformation().get(i).getFreespace() != null) {
1052                         com.setLong(3, req.getMachineData().getDriveInformation().get(i).getFreespace());
1053                     } else {
1054                         com.setLong(3, -1);
1055                     }
1056                     com.setLong(4, now);
1057 
1058                     req.getMachineData().getDriveInformation().get(i).setId(UUID.randomUUID().toString());
1059                     // com.setString(5, req.getMachineData().getDriveInformation().get(i).getId());
1060 
1061                     id = UUID.randomUUID();
1062                     req.getMachineData().setId(id.toString());
1063                     com.setString(5, id.toString());
1064                     if (req.getMachineData().getDriveInformation().get(i).getKilobytespersecondDiskWrite() != null) {
1065                         com.setLong(6, req.getMachineData().getDriveInformation().get(i).getKilobytespersecondDiskWrite());
1066                     } else {
1067                         com.setNull(6, java.sql.Types.BIGINT);
1068                     }
1069                     if (req.getMachineData().getDriveInformation().get(i).getKilobytespersecondDiskRead() != null) {
1070                         com.setLong(7, req.getMachineData().getDriveInformation().get(i).getKilobytespersecondDiskRead());
1071                     } else {
1072                         com.setNull(7, java.sql.Types.BIGINT);
1073                     }
1074                     com.setString(8, Utility.truncate(req.getMachineData().getDriveInformation().get(i).getSystemid(), MAXTEXT));
1075                     com.setBoolean(9, req.getMachineData().getDriveInformation().get(i).isOperational());
1076                     com.setString(10, Utility.truncate(req.getMachineData().getDriveInformation().get(i).getOperationalstatus(), MAXTEXT));
1077                     com.setString(11, Utility.truncate(req.getDomainname(), MAXTEXT));
1078                     com.setString(12, Utility.truncate(req.getMachineData().getUri(), MAXTEXT));
1079                     com.executeUpdate();
1080                 } catch (Exception ex) {
1081                     log.log(Level.ERROR, "DCS error saving drive information! " + ex.getMessage());
1082                     log.log(Level.DEBUG, "DCS error saving drive information! " + ex.getMessage(), ex);
1083                     failuresTX++;
1084                 } finally {
1085                     DBUtils.safeClose(com);
1086                 }
1087             }
1088         }
1089 
1090         if (!agents.containsKey(req.getAgentType())) {
1091             agents.put(req.getAgentType(), true);
1092             Connection config = Utility.getConfigurationDBConnection();
1093             try {
1094 
1095                 com = config.prepareStatement("INSERT INTO agents(agenttype)  VALUES (?)");
1096                 com.setString(1, Utility.truncate(req.getAgentType(), MAXTEXT));
1097                 com.execute();
1098 
1099                 log.log(Level.DEBUG, "DCS DEBUG adding agent type");
1100             } catch (Exception ex) {
1101                 log.log(Level.DEBUG, "DCS DEBUG adding agent type FAILED " + ex.getMessage());
1102 
1103             } finally {
1104                 DBUtils.safeClose(com);
1105                 DBUtils.safeClose(config);
1106             }
1107         }
1108 
1109         DBUtils.safeClose(con);
1110 
1111         if (!ok) {
1112             org.miloss.fgsms.services.interfaces.faults.ServiceUnavailableException t = new org.miloss.fgsms.services.interfaces.faults.ServiceUnavailableException();
1113             t.setCode(ServiceUnavailableFaultCodes.DATA_BASE_UNAVAILABLE);
1114             ServiceUnavailableException e = new ServiceUnavailableException("db unavailable", t);
1115             throw e;
1116         }
1117 
1118         AddMachineAndProcessDataResponseMsg res = new AddMachineAndProcessDataResponseMsg();
1119 
1120         res.setClassification(getCurrentClassificationLevel());
1121         //does all SLA alerts, change in status, policy create for everything BUT time based Rules
1122         try {
1123             MachinePolicy mp = SLACommon.ProcessMachineLevelSLAs(req, true);
1124             List<ProcessPolicy> pp = SLACommon.ProcessProcessLevelSLAs(req, true);
1125             res.setMachinePolicy(mp);
1126             res.getProcessPolicy().addAll(pp);
1127         } catch (Exception ex) {
1128             log.log(Level.ERROR, "error caught processing loading machine and process policies", ex);
1129         }
1130         
1131         if (res.getMachinePolicy()==null){
1132             //strange case, the os agent started (and is a new instance)
1133             //before the fgsms server was fullly up and running, so the initial
1134             //get policy message was never processed and thus the policy was never
1135             //created.
1136             //unfortunately making a new one here will cause some dependency spagetthi
1137             //so we'll just leave it as is and adjust the travis script to wait a bit
1138         }
1139 
1140         KeyNameValueEnc prop = DBSettingsLoader.GetPropertiesFromDB(true, "Agents.Process", "ReportingFrequency");
1141         if (prop == null || prop.getKeyNameValue() == null) {
1142             res.setReportingFrequency(df.newDuration(30000));
1143         } else {
1144             try {
1145                 res.setReportingFrequency(df.newDuration(Long.parseLong(prop.getKeyNameValue().getPropertyValue())));
1146             } catch (Exception ex) {
1147                 res.setReportingFrequency(df.newDuration(30000));
1148             }
1149         }
1150         successTX += req.getProcessData().size() + 1;
1151         return res;
1152 
1153     }
1154 
1155     /**
1156      * Get the operating status of this service
1157      *
1158      * @param request
1159      * @return returns
1160      * org.miloss.fgsms.services.interfaces.common.GetOperatingStatusResponseMessage
1161      */
1162     @WebMethod(operationName = "GetOperatingStatus", action = "urn:org:miloss:fgsms:services:interfaces:opStatusService/GetOperatingStatus")
1163     @WebResult(name = "response", targetNamespace = "urn:org:miloss:fgsms:services:interfaces:common")
1164     @RequestWrapper(localName = "GetOperatingStatus", targetNamespace = "urn:org:miloss:fgsms:services:interfaces:common", className = "org.miloss.fgsms.services.interfaces.common.GetOperatingStatus")
1165     @ResponseWrapper(localName = "GetOperatingStatusResponse", targetNamespace = "urn:org:miloss:fgsms:services:interfaces:common", className = "org.miloss.fgsms.services.interfaces.common.GetOperatingStatusResponse")
1166     public GetOperatingStatusResponseMessage getOperatingStatus(
1167             @WebParam(name = "request", targetNamespace = "urn:org:miloss:fgsms:services:interfaces:common") GetOperatingStatusRequestMessage request) {
1168         String currentUser = UserIdentityUtil.getFirstIdentityToString(ctx);
1169 
1170         Utility.validateClassification(request.getClassification());
1171         AuditLogger.logItem(this.getClass().getCanonicalName(), "getOperatingStatus", currentUser, "", (request.getClassification()), ctx.getMessageContext());
1172 
1173         GetOperatingStatusResponseMessage res = new GetOperatingStatusResponseMessage();
1174 
1175         res.setClassification(request.getClassification());
1176         res.setVersionInfo(new GetOperatingStatusResponseMessage.VersionInfo());
1177         res.getVersionInfo().setVersionData(org.miloss.fgsms.common.Constants.Version);
1178         res
1179                 .getVersionInfo().setVersionSource(org.miloss.fgsms.common.Constants.class
1180                         .getCanonicalName());
1181         res.setDataSentSuccessfully(successTX);
1182         res.setDataNotSentSuccessfully(failuresTX);
1183         res.setStartedAt(started);
1184         boolean ok = true;
1185         Connection con = Utility.getConfigurationDBConnection();
1186         Connection con2 = Utility.getPerformanceDBConnection();
1187         PreparedStatement prepareStatement = null;
1188         PreparedStatement prepareStatement2 = null;
1189         try {
1190             prepareStatement = con.prepareStatement("select 1=1;");
1191             prepareStatement.execute();
1192 
1193             prepareStatement2 = con2.prepareStatement("select 1=1;");
1194             prepareStatement2.execute();
1195 
1196             res.setStatusMessage("OK");
1197         } catch (Exception ex) {
1198             ok = false;
1199             res.setStatusMessage("One or more of the database connections is available");
1200         } finally {
1201             DBUtils.safeClose(prepareStatement);
1202             DBUtils.safeClose(prepareStatement2);
1203             DBUtils.safeClose(con);
1204             DBUtils.safeClose(con2);
1205         }
1206         res.setStatus(ok);
1207         successTX++;
1208         return res;
1209     }
1210 }