1
2
3
4
5
6 package org.miloss.fgsms.statistics.jobs;
7
8 import java.sql.Connection;
9 import java.sql.PreparedStatement;
10 import java.sql.ResultSet;
11 import java.sql.SQLException;
12 import java.util.ArrayList;
13 import java.util.Collections;
14 import java.util.HashMap;
15 import java.util.HashSet;
16 import java.util.List;
17 import java.util.Set;
18 import java.util.UUID;
19 import org.apache.log4j.Level;
20 import org.miloss.fgsms.agentcore.MessageProcessor;
21 import org.miloss.fgsms.common.DBUtils;
22 import org.miloss.fgsms.common.Utility;
23 import static org.miloss.fgsms.statistics.FgsmsStatsv2.SERVICE_NAME;
24 import static org.miloss.fgsms.statistics.FgsmsStatsv2.allitems;
25 import static org.miloss.fgsms.statistics.FgsmsStatsv2.log;
26 import static org.miloss.fgsms.statistics.FgsmsStatsv2.myUrl;
27 import org.miloss.fgsms.statistics.StatisticsContainer;
28
29
30
31
32
33 public class TransactionalStatisticsJob extends BaseJob {
34
35 List<Long> periods;
36 String uri;
37
38 public TransactionalStatisticsJob(List<Long> periods, String string) {
39 this.periods = periods;
40 this.uri = string;
41 }
42
43 @Override
44 public void run() {
45 UUID random = UUID.randomUUID();
46 MessageProcessor.getSingletonObject().processMessageInput(SERVICE_NAME + " for " + uri, 0, myUrl, "transactional", "system", random.toString(), new HashMap(), "", this.getClass().getCanonicalName(), "", "");
47 Connection ConfigCon = Utility.getConfigurationDBConnection();
48 Connection PerfCon = Utility.getPerformanceDBConnection();
49 try {
50 doWorkTransactional(ConfigCon, PerfCon, periods, uri);
51 MessageProcessor.getSingletonObject().processMessageOutput(random.toString(), "success", 0, false, System.currentTimeMillis(), new HashMap());
52 } catch (Exception ex) {
53 MessageProcessor.getSingletonObject().processMessageOutput(random.toString(), "error " + ex.getMessage(), 0, true, System.currentTimeMillis(), new HashMap());
54 log.log(Level.ERROR, null, ex);
55 } finally {
56 DBUtils.safeClose(PerfCon);
57 DBUtils.safeClose(ConfigCon);
58 }
59 }
60
61 private void createAllStats(List<StatisticsContainer> stats, List<Long> periods, final String url, final String action) {
62 for (int k = 0; k < periods.size(); k++) {
63 boolean found = false;
64 for (int i = 0; i < stats.size(); i++) {
65 if (stats.get(i).action.equals(action) && stats.get(i).uri.equals(url) && stats.get(i).timeperiod == periods.get(k).longValue()) {
66 found = true;
67 }
68 }
69 if (!found) {
70 StatisticsContainer s = new StatisticsContainer();
71 s.uri = url;
72 s.action = action;
73 s.timeperiod = periods.get(k);
74 stats.add(s);
75 }
76
77 }
78 }
79
80 private static long largestPeriod(List<Long> periods) {
81 long l = -1;
82 for (int i = 0; i < periods.size(); i++) {
83 if (periods.get(i).longValue() > l) {
84 l = periods.get(i).longValue();
85 }
86 }
87 return l;
88 }
89
90 private static long largestPeriod_PriceIsRightRules(List<Long> periods, final long timestamp) {
91 long l = -1;
92 for (int i = 0; i < periods.size(); i++) {
93
94 if (periods.get(i).longValue() >= (System.currentTimeMillis() - timestamp) && periods.get(i).longValue() > l) {
95 l = periods.get(i).longValue();
96 }
97 }
98 if (l == -1) {
99 return largestPeriod(periods);
100 }
101 return l;
102 }
103
104 private StatisticsContainer getStatsForTimeStamp(List<StatisticsContainer> stats, final Long timestamp, final String uri, final String action, final List<Long> periods) {
105 if (stats == null) {
106 stats = new ArrayList<StatisticsContainer>();
107 }
108
109
110 long eventHappenedNmsAgo = System.currentTimeMillis() - timestamp;
111
112
113 StatisticsContainer s = null;
114
115 for (int i = 0; i < stats.size(); i++) {
116
117 if (stats.get(i).uri.equals(uri)
118 && stats.get(i).action.equals(action)) {
119
120
121 if (s == null && eventHappenedNmsAgo < stats.get(i).timeperiod) {
122 s = stats.get(i);
123 } else if (s != null && stats.get(i).timeperiod < s.timeperiod && stats.get(i).timeperiod > eventHappenedNmsAgo) {
124 s = stats.get(i);
125 }
126
127 }
128 }
129 if (s == null) {
130 log.log(Level.DEBUG, "unexpected situation during statistics calculation");
131 s = new StatisticsContainer();
132 s.uri = uri;
133 s.action = action;
134 s.timeperiod = largestPeriod(periods);
135 stats.add(s);
136 }
137 return s;
138 }
139
140 private List<Long> addIfMissingMandatoryTimePeriods(List<Long> periods) {
141 periods.add(Long.valueOf(5 * 60 * 1000));
142 periods.add(Long.valueOf(15 * 60 * 1000));
143 periods.add(Long.valueOf(60 * 60 * 1000));
144 periods.add(Long.valueOf(24 * 60 * 60 * 1000));
145 Set<Long> minified = new HashSet<Long>(periods);
146 return new ArrayList<Long>(minified);
147 }
148
149 private void print(StatisticsContainer s) {
150 log.log(Level.DEBUG, s.timeperiod + " " + s.uri + " " + s.action + " " + s.success + " " + s.faults + " "
151 + s.averageresponsetime + " " + s.max_request_size + " " + s.max_response_size + " "
152 + s.max_responsetime + " " + s.mtbf + " " + s.availibity);
153
154 }
155
156 private void printPeriods(List<Long> periods) {
157 String s = "";
158 for (int i = 0; i < periods.size(); i++) {
159 s += periods.get(i) + " ";
160 }
161 log.log(Level.DEBUG, "calculating status using the following time periods " + s.trim());
162 }
163
164 private StatisticsContainer getStatsForTimePeriod(List<StatisticsContainer> stats, Long timeperiod, final String uri, final String action, List<Long> periods) {
165 if (stats == null) {
166 stats = new ArrayList<StatisticsContainer>();
167 }
168
169
170 StatisticsContainer s = null;
171 for (int i = 0; i < stats.size(); i++) {
172 if (stats.get(i).uri.equals(uri)
173 && stats.get(i).action.equals(action)
174 && stats.get(i).timeperiod == timeperiod) {
175 s = stats.get(i);
176 }
177 }
178 if (s == null) {
179 log.log(Level.WARN, "unexpected situation during statistics calculation, StatisticsContainer is null, i'll just create a new one and move on");
180 s = new StatisticsContainer();
181 s.uri = uri;
182 s.action = action;
183 s.timeperiod = largestPeriod(periods);
184 stats.add(s);
185 }
186 return s;
187 }
188
189 private void doWorkTransactional(Connection ConfigCon, Connection PerfCon, List<Long> periods, String uri) throws Exception {
190 PreparedStatement com = null;
191 long now = System.currentTimeMillis();
192
193 List<StatisticsContainer> stats = new ArrayList<StatisticsContainer>();
194 try {
195
196 log.log(Level.DEBUG, "loading actions for " + uri);
197 List<String> actions = new ArrayList<String>();
198 PreparedStatement comm = null;
199 ResultSet results = null;
200 try {
201 comm = PerfCon.prepareStatement("Select soapaction from actionlist where URI=? order by soapaction desc;");
202 comm.setString(1, uri);
203 results = comm.executeQuery();
204 while (results.next()) {
205 String temp = results.getString("soapaction");
206
207 log.log(Level.DEBUG, uri + " " + temp);
208 if (!Utility.stringIsNullOrEmpty(temp)) {
209 temp = temp.trim();
210 }
211 if (!Utility.stringIsNullOrEmpty(temp)) {
212 actions.add(temp);
213 }
214 }
215 } catch (Exception ex) {
216 log.error("error fetching actions for service", ex);
217 } finally {
218 DBUtils.safeClose(results);
219 DBUtils.safeClose(comm);
220 }
221 log.log(Level.INFO, "calculating statistics for " + uri);
222 actions.add(allitems);
223
224 for (int k = 0; k < actions.size(); k++) {
225
226 createAllStats(stats, periods, uri, actions.get(k));
227
228 for (int i = 0; i < periods.size(); i++) {
229 insertRow(PerfCon, uri, actions.get(k), periods.get(i));
230 }
231 }
232
233
234
235
236
237
238 PreparedStatement cmd = null;
239 ResultSet records = null;
240 try {
241 cmd = PerfCon.prepareStatement("select utcdatetime, success, uri,soapaction, responsetimems, slafault, requestsize, responsesize from rawdata where uri=? and utcdatetime > ?;");
242 cmd.setString(1, uri);
243 long limit = (long) ((long) now - largestPeriod(periods));
244 cmd.setLong(2, limit);
245 records = cmd.executeQuery();
246
247 while (records.next()) {
248 StatisticsContainer rollup = getStatsForTimeStamp(stats, records.getLong("utcdatetime"), records.getString("uri"), allitems, periods);
249 StatisticsContainer s = getStatsForTimeStamp(stats, records.getLong("utcdatetime"), records.getString("uri"), records.getString("soapaction"), periods);
250 if (records.getBoolean("success")) {
251 s.success++;
252 rollup.success++;
253 } else {
254 s.faults++;
255 rollup.faults++;
256 }
257 if (!Utility.stringIsNullOrEmpty(records.getString("slafault"))) {
258 s.sla++;
259
260 }
261 s.totalprocessingtime += records.getInt("responsetimems");
262 rollup.totalprocessingtime += records.getInt("responsetimems");
263 if (records.getInt("requestsize") > s.max_request_size) {
264 s.max_request_size = records.getInt("requestsize");
265 }
266 if (records.getInt("responsesize") > s.max_response_size) {
267 s.max_response_size = records.getInt("responsesize");
268 }
269 if (records.getInt("responsetimems") > s.max_responsetime) {
270 s.max_responsetime = records.getInt("responsetimems");
271 }
272
273 if (records.getInt("requestsize") > rollup.max_request_size) {
274 rollup.max_request_size = records.getInt("requestsize");
275 }
276 if (records.getInt("responsesize") > rollup.max_response_size) {
277 rollup.max_response_size = records.getInt("responsesize");
278 }
279 if (records.getInt("responsetimems") > rollup.max_responsetime) {
280 rollup.max_responsetime = records.getInt("responsetimems");
281 }
282
283 }
284 } catch (Exception ex) {
285 log.error("error fetching transactions", ex);
286 } finally {
287 DBUtils.safeClose(records);
288 DBUtils.safeClose(cmd);
289 }
290
291
292 for (int i = 0; i < periods.size(); i++) {
293 StatisticsContainer rollup = getStatsForTimePeriod(stats, periods.get(i), uri, allitems, periods);
294 long slafaults = getSLACount(uri, periods.get(i), PerfCon);
295 rollup.sla = slafaults;
296 rollup.availibity = getAvailability(now, periods.get(i), uri, allitems, PerfCon, ConfigCon);
297 rollup.mtbf = getMTBF(System.currentTimeMillis(), rollup.timeperiod, rollup.uri, rollup.action, PerfCon);
298 for (int k = 0; k < actions.size(); k++) {
299 StatisticsContainer s = getStatsForTimePeriod(stats, periods.get(i), uri, actions.get(k), periods);
300 s.availibity = rollup.availibity;
301 s.mtbf = rollup.mtbf = getMTBF(System.currentTimeMillis(), rollup.timeperiod, rollup.uri, rollup.action, PerfCon);
302 }
303 }
304
305 for (int i = 0; i < stats.size(); i++)
306 {
307 print(stats.get(i));
308 insertRow(PerfCon, stats.get(i).uri, stats.get(i).action, stats.get(i).timeperiod);
309 PreparedStatement up = null;
310 try {
311 up = PerfCon.prepareStatement("UPDATE agg2 set "
312 + "success=?, failure=?, avgres=?, avail=?, sla=?, mtbf=?, maxreq=?, maxres=?, maxresponsetime=?, "
313 + " timestampepoch=? "
314 + "WHERE uri=? and soapaction=? and timerange=?;");
315 up.setLong(1, stats.get(i).success);
316 up.setLong(2, stats.get(i).faults);
317 if (stats.get(i).success + stats.get(i).faults == 0) {
318 up.setLong(3, 0);
319 } else {
320 up.setLong(3, (long) (stats.get(i).totalprocessingtime / (long) ((long) stats.get(i).success + (long) stats.get(i).faults)));
321 }
322 if (stats.get(i).availibity < 0) {
323 up.setDouble(4, 0.0);
324 } else {
325 up.setDouble(4, stats.get(i).availibity);
326 }
327 up.setLong(5, stats.get(i).sla);
328 up.setLong(6, stats.get(i).mtbf);
329 up.setLong(7, stats.get(i).max_request_size);
330 up.setLong(8, stats.get(i).max_response_size);
331 up.setLong(9, stats.get(i).max_responsetime);
332 up.setLong(10, System.currentTimeMillis());
333 up.setString(11, stats.get(i).uri);
334 up.setString(12, stats.get(i).action);
335 up.setLong(13, stats.get(i).timeperiod);
336 up.executeUpdate();
337 } catch (Exception ex) {
338 log.error("error updating statistcs row", ex);
339 } finally {
340 DBUtils.safeClose(up);
341 }
342
343 }
344 now = System.currentTimeMillis()-now;
345
346 log.log(Level.INFO, "Statistics calculations took " + uri + " " + now + "ms");
347 } catch (Exception ex) {
348 log.log(Level.ERROR, "Error caught during statistics calculation, please report", ex);
349 throw ex;
350 } finally {
351
352 DBUtils.safeClose(com);
353 }
354 }
355
356 private long getMTBF(final long now, final long limit, final String url, final String action, Connection PerfCon) {
357 PreparedStatement prepareStatement = null;
358 ResultSet rs = null;
359 try {
360 List<Long> faults = new ArrayList<Long>();
361
362 if (action.equals(allitems)) {
363 prepareStatement = PerfCon.prepareStatement("select utcdatetime from rawdata where success=false and uri=? and utcdatetime > ?;");
364 prepareStatement.setString(1, url);
365 prepareStatement.setLong(2, now - limit);
366 } else {
367 prepareStatement = PerfCon.prepareStatement("select utcdatetime from rawdata where success=false and action=? and uri=? and utcdatetime > ?;");
368 prepareStatement.setString(1, action);
369 prepareStatement.setString(2, url);
370 prepareStatement.setLong(3, now - limit);
371 }
372 rs = prepareStatement.executeQuery();
373 while (rs.next()) {
374 faults.add(rs.getLong(1));
375 }
376 rs.close();
377
378 long diff = 0;
379 Collections.sort(faults);
380 if (faults.size() < 2) {
381 return -1;
382 }
383 for (int i = 0; i < faults.size() - 1; i++) {
384 diff += faults.get(i + 1) - faults.get(i);
385 }
386 return diff / (faults.size() - 1);
387 } catch (SQLException ex) {
388 log.log(Level.WARN, null, ex);
389 } finally {
390 DBUtils.safeClose(rs);
391 DBUtils.safeClose(prepareStatement);
392 }
393 return 0;
394
395 }
396
397 }