4 // Fraunhofer Institute for Open Communication Systems (FOKUS)
5 // Competence Center NETwork research (NET), St. Augustin, GERMANY
6 // Stefan Bund <g0dil@berlios.de>
8 // This program is free software; you can redistribute it and/or modify
9 // it under the terms of the GNU General Public License as published by
10 // the Free Software Foundation; either version 2 of the License, or
11 // (at your option) any later version.
13 // This program is distributed in the hope that it will be useful,
14 // but WITHOUT ANY WARRANTY; without even the implied warranty of
15 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 // GNU General Public License for more details.
18 // You should have received a copy of the GNU General Public License
19 // along with this program; if not, write to the
20 // Free Software Foundation, Inc.,
21 // 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
24 \brief Statistics public header */
26 #ifndef HH_SENF_Utils_Statistics_
27 #define HH_SENF_Utils_Statistics_ 1
33 #include <boost/iterator/transform_iterator.hpp>
34 #include <boost/range/iterator_range.hpp>
35 #include <boost/utility.hpp>
36 #include <boost/ptr_container/ptr_vector.hpp>
37 #include <boost/signals.hpp>
38 #include <senf/Utils/Logger/Logger.hh>
39 #include <senf/Utils/Console/ScopedDirectory.hh>
40 #include "StatisticAccumulator.hh"
41 #include "Exception.hh"
43 //#include "Statistics.mpp"
44 //-/////////////////////////////////////////////////////////////////////////////////////////////////
48 /** \defgroup senf_statistics Statistics
50 The statistics functionality has two parts:
52 \li the senf::Statistics class
53 \li the senf::StatisticsAccumulator class
54 \li statistics sources
56 Each senf::Statistics instance collects information about a single parameter. Which
57 parameter is set up by connecting the Statistics instance with an arbitrary statistics
60 %Statistics sources are <a href="http://www.boost.org/doc/libs/release/doc/html/signals.html">
61 Boost Signals</a> which are emitted periodically to provide new data.
67 /** \brief Internal: Generic Statistics collection */
70 typedef std::map<unsigned, Collector> Children;
73 typedef Children::value_type & first_argument_type;
74 typedef Collector & result_type;
75 result_type operator()(first_argument_type i) const;
78 typedef boost::transform_iterator<Transform,Children::iterator> ValueIterator;
83 //-////////////////////////////////////////////////////////////////////////
86 typedef boost::iterator_range<ValueIterator> CollectorRange;
88 /** \brief Output connection interface
90 This class is returned from senf::StatisticsBase::output() and the derived
91 <tt>output()</tt> implementations to allow connecting an output with an arbitrary
94 There are two types of targets:
95 \li <em>Externally managed targets</em>. These targets live outside of the statistics
96 module, just a reference to those targets is saved. The target should derive from
97 <tt>boost::signals::trackable</tt> to ensure they are automatically disconnected
99 \li <em>Internally managed targets</em>. Those targets are owned by the statistics
100 module and will be destroyed when the statistics class is destroyed.
102 Externally managed targets are passed by non-const reference to connect(), internally
103 managed targets are passed using <tt>std::auto_ptr</tt>.
105 A target is any callable object which takes three float values as argument: The current
106 minimum, average and maximum value.
109 // Simple function as statistics target
110 void collect(float min, float avg, float max)
116 void operator()(float min, float avg, float max, float dev)
121 \ingroup senf_statistics
123 template <class Owner>
127 template <class Target> Owner & connect(Target & target,
128 std::string label="") const;
129 ///< Connect externally managed target
130 template <class PTarget> Owner & connect(std::auto_ptr<PTarget> target,
131 std::string label="") const;
132 ///< Connect internally managed target
133 Owner & noconnect() const; ///< Don't connect the output
134 senf::console::ScopedDirectory<> & dir() const;
135 ///< Get target's console directory
140 OutputProxy(Owner * owner, OutputEntry * entry);
141 template <class OtherOwner>
142 OutputProxy(Owner * owner, OutputProxy<OtherOwner> const & other);
146 OutputEntry * entry_;
148 template <class OtherOwner> friend class OutputProxy;
151 //-////////////////////////////////////////////////////////////////////////
152 ///\name Accessing the current value
155 float min() const; ///< Last min value entered
156 float avg() const; ///< Last avg value entered
157 float max() const; ///< Last max value entered
158 float dev() const; ///< Last dev value entered
160 virtual unsigned rank() const; ///< Return collectors rank value
161 /**< \returns number of basic values collected into each new
162 value by this collector. */
165 //-////////////////////////////////////////////////////////////////////////
166 ///\name Child collectors
169 Collector & operator[](unsigned rank);
170 ///< Get child collector
171 /**< This member will return a reference to the collector
172 collecting \a rank values.
173 \param[in] rank Number of values the requested
174 collector collects into each combined value.
175 \throws InvalidRankException if \a rank is not a valid
176 registered rank value. */
177 Collector const & operator[](unsigned rank) const;
178 ///< Get child collector
179 /**< This member will return a const reference to the
180 collector collecting \a rank values.
181 \param[in] rank Number of values the requested
182 collector collects into each combined value.
183 \throws InvalidRankException if \a rank is not a valid
184 registered rank value. */
185 CollectorRange collectors(); ///< List all child collectors
186 /**< \returns iterator range of child collector
189 Collector & collect(unsigned rank); ///< Register a new collector
190 /**< Adds a collector collecting \a rank values into each
192 \param[in] rank number of values to collect
193 \returns Reference to new collector
194 \throws DuplicateRankException if a collector
195 collecting \a rank values already exists. */
197 Statistics & base(); ///< Get base statistics object
198 /**< Returns the base statistics object. If this is
199 a child collector, this will return the outermost
200 statistics object, otherwise it will return
203 std::string path() const; ///< Get the path to this collector
204 /**< Returns the '-'-separated list of collectors up to
205 here. If this is the basic statistics object, the value
206 is empty, otherwise it is built by joining the
210 //-////////////////////////////////////////////////////////////////////////
211 ///\name Result generation
213 OutputProxy<StatisticsBase> output(unsigned n = 1u);
215 /**< This call will request the collector to output
216 statistics build by averaging the last \a n
217 values. This output is generated for every new value in
218 the collector. The output signal can be connected to an
219 arbitrary target using the returned proxy. Example:
221 stats.output(4u).connect(
222 senf::StatisticsLogger());
224 \param[in] n size of sliding average window */
227 //-////////////////////////////////////////////////////////////////////////
230 struct InvalidRankException : public senf::Exception
231 { InvalidRankException() : senf::Exception("Invalid rank value") {} };
233 struct DuplicateRankException : public senf::Exception
234 { DuplicateRankException() : senf::Exception("Duplicate rank value") {} };
236 void consoleList(unsigned level, std::ostream & os) const;
240 virtual ~StatisticsBase();
241 void enter(unsigned n, float min, float avg, float max, float dev);
244 virtual Statistics & v_base() = 0;
245 virtual std::string v_path() const = 0;
247 void generateOutput();
260 QueueEntry() : min(), avg(), max(), dev() {}
261 QueueEntry(float min_, float avg_, float max_, float dev_)
262 : min(min_), avg(avg_), max(max_), dev(dev_) {}
264 typedef std::deque<QueueEntry> Queue;
270 explicit TargetBase(std::string const & label_) : label (label_) {}
271 virtual ~TargetBase() {};
275 template <class PTarget>
276 struct Target : public TargetBase
278 boost::scoped_ptr<PTarget> target_;
279 Target(std::auto_ptr<PTarget> target, std::string const & label)
280 : TargetBase (label), target_ (target.release()) {}
281 explicit Target(std::string const & label)
282 : TargetBase (label), target_ (0) {}
286 explicit OutputEntry(unsigned n_);
287 OutputEntry(const OutputEntry& other);
288 OutputEntry& operator=(const OutputEntry& other);
291 void consoleList(std::ostream & os);
299 boost::signal<void(float,float,float,float)> signal;
300 boost::ptr_vector<TargetBase> targets_;
302 senf::console::ScopedDirectory<> dir;
304 typedef std::map<unsigned, OutputEntry> OutputMap;
306 unsigned maxQueueLen_;
309 /** \brief Collect statistics and generate %log messages
311 The Statistics class collects information about a single value. The assumption is, that
312 Statistics::operator() is called \e periodically to supply new data.
314 Each data entry is comprised of a minimum, average and maximum value. These values should
315 describe the value of whatever parameter is analyzed by a Statistics instance over the last
316 period. The length of the period is defined by the interval, at which data is entered.
318 The Statistics class combines successive data values into ever larger groups. These groups
319 form a tree where each node combines a fixed number of data values of it's parent node. An
322 Let us assume, that Statistics::operator() is called every 100ms. We can than build a chain
324 \li The basic statistics module provides information with a resolution of 1/10th of a second
325 \li A collector collecting 10 values provides information with 1 second resolution
326 \li The next collector collects 60 values and provides a resolution of 1 minute
327 \li Again the next collector collects 60 values and provides a resolution of 1 hour
330 This way, we create a hierarchy of values. Each collector manages a minimum, average and
331 maximum value always created over it's the last complete interval.
333 It is possible to have more than one collector based on the same basic interval. In above
334 scenario, we might want to add another collector which for example collects information
335 with a 100 second scale. This is possible and changes the list of collectors into a tree.
337 Now to turn all this into code:
340 senf::ppi::module::RateAnalyzer rateAnalyzer;
341 senf::Statistics packetStats;
343 rateAnalyzer.signals.packetsPerSecond.connect(boost::ref(packetStats));
345 packetStats // called 10 times / second
346 .collect(10u) // seconds
347 .collect(60u) // minutes
348 .collect(60u); // hours
350 packetStats[10u].collect(100u); // 100 seconds
352 rateAnalyzer.startStatistics(senf::ClockService::milliseconds(100u));
355 This code will collect the statistics as described in the Example above. However, no output
358 For every collector, any number of outputs may be defined. Each output consists of the
359 number of values to calculate a sliding average over and an identifying label.
361 Lets say, we want to produce the following outputs:
362 \li A sliding average of 5 values based on the raw 1/10th second data.
363 \li Three different outputs from the seconds staistics: current value without average,
364 sliding average over 10 seconds and sliding average over 60 seconds.
365 \li Output the minutes and hourly value without averaging.
367 To achieve this, we can augment above code in the following way:
371 .output( 5u).connect(senf::StatisicsLogger("pps 100ms 5"))
373 .output( ).noconnect()
374 .output(10u).connect(senf::StatisicsLogger("pps 1s 10"))
375 .output(60u).connect(senf::StatisicsLogger("pps 1s 60"))
377 .output( ).connect(senf::StatisicsLogger("pps 1min 1"))
379 .output( ).connect(senf::StatisicsLogger("pps 1h 1"));
381 packetStats.output(5u).connect(
382 senf::StatisticsLogger<senf::log::Debug, senf::log::VERBOSE>("pps"));
384 senf::log::FileTarget statslog ("stats.log");
386 statslog.showArea(false);
387 statslog.showLevel(false);
389 statslog.timeFormat("");
391 statslog.route<senf::StatisticsStream>();
394 We use a StatisticsLogger to send the %log messages to the senf::StatisticsStream %log
395 stream. The stream, area an level to send the statistics %log messages to may be configured
396 using template arguments to StatisticsLogger.
398 It is also possible to skip sending the output to any target or send one output to several
401 Here we have opted to use a label which explicitly describes the name of the variable, the
402 basic interval and the size of the sliding average window. However, the label is completely
405 All output is generated using the Senf Logger on the senf::StatisticsStream %log stream. In
406 the example above, we have configured a special logfile \c stats.log which contains the
407 statistics values each prefixed with a timestamp in nanoseconds (but no further information
408 like %log level or tag):
410 0000000000.000000000 pps 100ms 5 43.3 43.3 43.3
411 0000000000.010311928 pps 100ms 5 42.5 42.5 42.5
413 0000000001.002413391 pps 100ms 5 62.0 62.0 62.0
414 0000000001.003920018 pps 1s 1 42.1 53.4 62.0
417 (the nanosecond values will of course be somewhat different ...)
419 \see senf::StatisticsBase::OutputProxy for the output proxy (connect) interface
420 \ingroup senf_statistics
423 : public StatisticsBase, boost::noncopyable
426 #ifndef SENF_DISABLE_CONSOLE
427 console::ScopedDirectory<Statistics> dir;
432 void operator()(unsigned n, float min, float avg, float max, float dev);
434 /**< This member must be called whenever new data is
437 If \a min and \a max values are not available, this
438 member should be called with \a min, \a avg and \a max
439 set to the same value. If no error estimate is
440 available, call with \a dev = 0.
442 In the most common case, this member is to be called
443 periodically and \a n will be 1 on all calls. The
444 interval, at which this member is called then defines
445 the statistics time scale.
447 Calling with \a n > 1 will submit the value \a n
448 times. This makes it possible to aggregate multiple
449 time slices into a single call. This does not change
450 the basic time scale but can change the number of
451 submits per unit time. If the basic time slice is
452 small, this allows to submit values almost arbitrarily
455 \param[in] n number of time-slices
456 \param[in] min minimal data value since last call
457 \param[in] avg average data value since last call
458 \param[in] max maximal data values since last call
459 \param[in] dev standard deviation of avg value */
460 void operator()(float min, float avg, float max, float dev=0.0f);
461 ///< Same as operator() with \a n==1
462 /**< Provided so a Statistics instance can be directly used
463 as a signal target. */
464 void operator()(float value, float dev=0.0f);
465 ///< Same as operator() with \a min == \a avg == \a max
466 /**< Provided so a Statistics instance can be directly used
467 as a signal target. */
468 void operator()(StatisticsData const & data);
469 ///< Same as operator(), but imports statistics data from a StatisticsData object
470 /**< Provided so a Statistics instance can be directly used
471 as a signal target. */
472 template <class Value>
473 void operator()(unsigned n, StatisticAccumulator<Value> & sa);
474 ///< Same as operator() gathers values from StatisticAccumulator
475 /**< Provided so a Statistics instance can be directly used
476 as a signal target. Caution: Clears values in
477 StatisticAccumulator afterwards
478 \param[in] n number of time-slices
479 \param[in] sa StatisticAccumulator*/
481 StatisticsBase::OutputProxy<Statistics> output(unsigned n = 1u);
483 void consoleList(std::ostream & os);
484 void consoleCollect(std::vector<unsigned> & ranks);
485 boost::shared_ptr<senf::console::DirectoryNode> consoleOutput(
486 std::vector<unsigned> & ranks, unsigned window);
489 Statistics & v_base();
490 std::string v_path() const;
493 /** \brief Accumulated statistics collector
495 This class collects accumulated statistics. It is automatically created by
496 senf::Statistics::collect()
498 \see senf::Statistics
500 class Collector : public StatisticsBase
503 virtual unsigned rank() const;
505 StatisticsBase::OutputProxy<Collector> output(unsigned n = 1u);
508 Collector(StatisticsBase * owner, unsigned rank);
509 void enter(unsigned n, float min, float avg, float max, float dev);
510 Statistics & v_base();
511 std::string v_path() const;
519 StatisticsBase * owner_;
521 friend class StatisticsBase;
526 //-/////////////////////////////////////////////////////////////////////////////////////////////////
527 #include "Statistics.cci"
528 //#include "Statistics.ct"
529 #include "Statistics.cti"
536 // comment-column: 40
537 // c-file-style: "senf"
538 // indent-tabs-mode: nil
539 // ispell-local-dictionary: "american"
540 // compile-command: "scons -u test"