4 // Fraunhofer Institute for Open Communication Systems (FOKUS)
6 // The contents of this file are subject to the Fraunhofer FOKUS Public License
7 // Version 1.0 (the "License"); you may not use this file except in compliance
8 // with the License. You may obtain a copy of the License at
9 // http://senf.berlios.de/license.html
11 // The Fraunhofer FOKUS Public License Version 1.0 is based on,
12 // but modifies the Mozilla Public License Version 1.1.
13 // See the full license text for the amendments.
15 // Software distributed under the License is distributed on an "AS IS" basis,
16 // WITHOUT WARRANTY OF ANY KIND, either express or implied. See the License
17 // for the specific language governing rights and limitations under the License.
19 // The Original Code is Fraunhofer FOKUS code.
21 // The Initial Developer of the Original Code is Fraunhofer-Gesellschaft e.V.
22 // (registered association), Hansastraße 27 c, 80686 Munich, Germany.
23 // All Rights Reserved.
26 // Stefan Bund <g0dil@berlios.de>
29 \brief Statistics public header */
31 #ifndef HH_SENF_Utils_Statistics_
32 #define HH_SENF_Utils_Statistics_ 1
38 #include <boost/iterator/transform_iterator.hpp>
39 #include <boost/range/iterator_range.hpp>
40 #include <boost/utility.hpp>
41 #include <boost/ptr_container/ptr_vector.hpp>
42 #include <boost/signals.hpp>
43 #include <senf/Utils/Logger/Logger.hh>
44 #include <senf/Utils/Console/ScopedDirectory.hh>
45 #include "StatisticAccumulator.hh"
46 #include "Exception.hh"
48 //#include "Statistics.mpp"
49 //-/////////////////////////////////////////////////////////////////////////////////////////////////
53 /** \defgroup senf_statistics Statistics
55 The statistics functionality has two parts:
57 \li the senf::Statistics class
58 \li the senf::StatisticsAccumulator class
59 \li statistics sources
61 Each senf::Statistics instance collects information about a single parameter. Which
62 parameter is set up by connecting the Statistics instance with an arbitrary statistics
65 %Statistics sources are <a href="http://www.boost.org/doc/libs/release/doc/html/signals.html">
66 Boost Signals</a> which are emitted periodically to provide new data.
72 /** \brief Internal: Generic Statistics collection */
75 typedef std::map<unsigned, Collector> Children;
77 // why we can't use ::__gnu_cxx::select2nd instead?!?!
79 typedef Children::value_type & first_argument_type;
80 typedef Collector & result_type;
81 result_type operator()(first_argument_type i) const;
84 typedef boost::transform_iterator<Transform,Children::iterator> ValueIterator;
89 //-////////////////////////////////////////////////////////////////////////
92 typedef boost::iterator_range<ValueIterator> CollectorRange;
94 /** \brief Output connection interface
96 This class is returned from senf::StatisticsBase::output() and the derived
97 <tt>output()</tt> implementations to allow connecting an output with an arbitrary
100 There are two types of targets:
101 \li <em>Externally managed targets</em>. These targets live outside of the statistics
102 module, just a reference to those targets is saved. The target should derive from
103 <tt>boost::signals::trackable</tt> to ensure they are automatically disconnected
105 \li <em>Internally managed targets</em>. Those targets are owned by the statistics
106 module and will be destroyed when the statistics class is destroyed.
108 Externally managed targets are passed by non-const reference to connect(), internally
109 managed targets are passed using <tt>std::auto_ptr</tt>.
111 A target is any callable object which takes three float values as argument: The current
112 minimum, average and maximum value.
115 // Simple function as statistics target
116 void collect(float min, float avg, float max)
122 void operator()(float min, float avg, float max, float dev)
127 \ingroup senf_statistics
129 template <class Owner>
133 template <class Target> Owner & connect(Target & target,
134 std::string label="") const;
135 ///< Connect externally managed target
136 template <class PTarget> Owner & connect(std::auto_ptr<PTarget> target,
137 std::string label="") const;
138 ///< Connect internally managed target
139 Owner & noconnect() const; ///< Don't connect the output
140 senf::console::ScopedDirectory<> & dir() const;
141 ///< Get target's console directory
146 OutputProxy(Owner * owner, OutputEntry * entry);
147 template <class OtherOwner>
148 OutputProxy(Owner * owner, OutputProxy<OtherOwner> const & other);
152 OutputEntry * entry_;
154 template <class OtherOwner> friend class OutputProxy;
157 //-////////////////////////////////////////////////////////////////////////
158 ///\name Accessing the current value
161 float min() const; ///< Last min value entered
162 float avg() const; ///< Last avg value entered
163 float max() const; ///< Last max value entered
164 float dev() const; ///< Last dev value entered
166 virtual unsigned rank() const; ///< Return collectors rank value
167 /**< \returns number of basic values collected into each new
168 value by this collector. */
171 //-////////////////////////////////////////////////////////////////////////
172 ///\name Child collectors
175 Collector & operator[](unsigned rank);
176 ///< Get child collector
177 /**< This member will return a reference to the collector
178 collecting \a rank values.
179 \param[in] rank Number of values the requested
180 collector collects into each combined value.
181 \throws InvalidRankException if \a rank is not a valid
182 registered rank value. */
183 Collector const & operator[](unsigned rank) const;
184 ///< Get child collector
185 /**< This member will return a const reference to the
186 collector collecting \a rank values.
187 \param[in] rank Number of values the requested
188 collector collects into each combined value.
189 \throws InvalidRankException if \a rank is not a valid
190 registered rank value. */
191 CollectorRange collectors(); ///< List all child collectors
192 /**< \returns iterator range of child collector
195 Collector & collect(unsigned rank); ///< Register a new collector
196 /**< Adds a collector collecting \a rank values into each
198 \param[in] rank number of values to collect
199 \returns Reference to new collector
200 \throws DuplicateRankException if a collector
201 collecting \a rank values already exists. */
203 Statistics & base(); ///< Get base statistics object
204 /**< Returns the base statistics object. If this is
205 a child collector, this will return the outermost
206 statistics object, otherwise it will return
209 std::string path() const; ///< Get the path to this collector
210 /**< Returns the '-'-separated list of collectors up to
211 here. If this is the basic statistics object, the value
212 is empty, otherwise it is built by joining the
216 //-////////////////////////////////////////////////////////////////////////
217 ///\name Result generation
219 OutputProxy<StatisticsBase> output(unsigned n = 1u);
221 /**< This call will request the collector to output
222 statistics build by averaging the last \a n
223 values. This output is generated for every new value in
224 the collector. The output signal can be connected to an
225 arbitrary target using the returned proxy. Example:
227 stats.output(4u).connect(
228 senf::StatisticsLogger());
230 \param[in] n size of sliding average window */
233 StatisticsData data() const; ///< Get the Statistics data as senf::StatisticsData
234 /**< Return a Statistic Data object containing values
235 from this instance. */
238 //-////////////////////////////////////////////////////////////////////////
241 struct InvalidRankException : public senf::Exception
242 { InvalidRankException() : senf::Exception("Invalid rank value") {} };
244 struct DuplicateRankException : public senf::Exception
245 { DuplicateRankException() : senf::Exception("Duplicate rank value") {} };
247 void consoleList(unsigned level, std::ostream & os) const;
251 virtual ~StatisticsBase();
252 void enter(unsigned n, float min, float avg, float max, float dev);
255 virtual Statistics & v_base() = 0;
256 virtual std::string v_path() const = 0;
258 void generateOutput();
271 QueueEntry() : min(), avg(), max(), dev() {}
272 QueueEntry(float min_, float avg_, float max_, float dev_)
273 : min(min_), avg(avg_), max(max_), dev(dev_) {}
275 typedef std::deque<QueueEntry> Queue;
281 explicit TargetBase(std::string const & label_) : label (label_) {}
282 virtual ~TargetBase() {};
286 template <class PTarget>
287 struct Target : public TargetBase
289 boost::scoped_ptr<PTarget> target_;
290 Target(std::auto_ptr<PTarget> target, std::string const & label)
291 : TargetBase (label), target_ (target.release()) {}
292 explicit Target(std::string const & label)
293 : TargetBase (label), target_ (0) {}
297 explicit OutputEntry(unsigned n_);
298 OutputEntry(const OutputEntry& other);
299 OutputEntry& operator=(const OutputEntry& other);
302 void consoleList(std::ostream & os);
310 boost::signal<void(float,float,float,float)> signal;
311 boost::ptr_vector<TargetBase> targets_;
313 senf::console::ScopedDirectory<> dir;
315 typedef std::map<unsigned, OutputEntry> OutputMap;
317 unsigned maxQueueLen_;
320 /** \brief Collect statistics and generate %log messages
322 The Statistics class collects information about a single value. The assumption is, that
323 Statistics::operator() is called \e periodically to supply new data.
325 Each data entry is comprised of a minimum, average and maximum value. These values should
326 describe the value of whatever parameter is analyzed by a Statistics instance over the last
327 period. The length of the period is defined by the interval, at which data is entered.
329 The Statistics class combines successive data values into ever larger groups. These groups
330 form a tree where each node combines a fixed number of data values of it's parent node. An
333 Let us assume, that Statistics::operator() is called every 100ms. We can than build a chain
335 \li The basic statistics module provides information with a resolution of 1/10th of a second
336 \li A collector collecting 10 values provides information with 1 second resolution
337 \li The next collector collects 60 values and provides a resolution of 1 minute
338 \li Again the next collector collects 60 values and provides a resolution of 1 hour
341 This way, we create a hierarchy of values. Each collector manages a minimum, average and
342 maximum value always created over it's the last complete interval.
344 It is possible to have more than one collector based on the same basic interval. In above
345 scenario, we might want to add another collector which for example collects information
346 with a 100 second scale. This is possible and changes the list of collectors into a tree.
348 Now to turn all this into code:
351 senf::ppi::module::RateAnalyzer rateAnalyzer;
352 senf::Statistics packetStats;
354 rateAnalyzer.signals.packetsPerSecond.connect(boost::ref(packetStats));
356 packetStats // called 10 times / second
357 .collect(10u) // seconds
358 .collect(60u) // minutes
359 .collect(60u); // hours
361 packetStats[10u].collect(100u); // 100 seconds
363 rateAnalyzer.startStatistics(senf::ClockService::milliseconds(100u));
366 This code will collect the statistics as described in the Example above. However, no output
369 For every collector, any number of outputs may be defined. Each output consists of the
370 number of values to calculate a sliding average over and an identifying label.
372 Lets say, we want to produce the following outputs:
373 \li A sliding average of 5 values based on the raw 1/10th second data.
374 \li Three different outputs from the seconds staistics: current value without average,
375 sliding average over 10 seconds and sliding average over 60 seconds.
376 \li Output the minutes and hourly value without averaging.
378 To achieve this, we can augment above code in the following way:
382 .output( 5u).connect(senf::StatisicsLogger("pps 100ms 5"))
384 .output( ).noconnect()
385 .output(10u).connect(senf::StatisicsLogger("pps 1s 10"))
386 .output(60u).connect(senf::StatisicsLogger("pps 1s 60"))
388 .output( ).connect(senf::StatisicsLogger("pps 1min 1"))
390 .output( ).connect(senf::StatisicsLogger("pps 1h 1"));
392 packetStats.output(5u).connect(
393 senf::StatisticsLogger<senf::log::Debug, senf::log::VERBOSE>("pps"));
395 senf::log::FileTarget statslog ("stats.log");
397 statslog.showArea(false);
398 statslog.showLevel(false);
400 statslog.timeFormat("");
402 statslog.route<senf::StatisticsStream>();
405 We use a StatisticsLogger to send the %log messages to the senf::StatisticsStream %log
406 stream. The stream, area an level to send the statistics %log messages to may be configured
407 using template arguments to StatisticsLogger.
409 It is also possible to skip sending the output to any target or send one output to several
412 Here we have opted to use a label which explicitly describes the name of the variable, the
413 basic interval and the size of the sliding average window. However, the label is completely
416 All output is generated using the Senf Logger on the senf::StatisticsStream %log stream. In
417 the example above, we have configured a special logfile \c stats.log which contains the
418 statistics values each prefixed with a timestamp in nanoseconds (but no further information
419 like %log level or tag):
421 0000000000.000000000 pps 100ms 5 43.3 43.3 43.3
422 0000000000.010311928 pps 100ms 5 42.5 42.5 42.5
424 0000000001.002413391 pps 100ms 5 62.0 62.0 62.0
425 0000000001.003920018 pps 1s 1 42.1 53.4 62.0
428 (the nanosecond values will of course be somewhat different ...)
430 \see senf::StatisticsBase::OutputProxy for the output proxy (connect) interface
431 \ingroup senf_statistics
434 : public StatisticsBase, boost::noncopyable
437 #ifndef SENF_DISABLE_CONSOLE
438 console::ScopedDirectory<Statistics> dir;
443 void operator()(unsigned n, float min, float avg, float max, float dev);
445 /**< This member must be called whenever new data is
448 If \a min and \a max values are not available, this
449 member should be called with \a min, \a avg and \a max
450 set to the same value. If no error estimate is
451 available, call with \a dev = 0.
453 In the most common case, this member is to be called
454 periodically and \a n will be 1 on all calls. The
455 interval, at which this member is called then defines
456 the statistics time scale.
458 Calling with \a n > 1 will submit the value \a n
459 times. This makes it possible to aggregate multiple
460 time slices into a single call. This does not change
461 the basic time scale but can change the number of
462 submits per unit time. If the basic time slice is
463 small, this allows to submit values almost arbitrarily
466 \param[in] n number of time-slices
467 \param[in] min minimal data value since last call
468 \param[in] avg average data value since last call
469 \param[in] max maximal data values since last call
470 \param[in] dev standard deviation of avg value */
471 void operator()(float min, float avg, float max, float dev=0.0f);
472 ///< Same as operator() with \a n==1
473 /**< Provided so a Statistics instance can be directly used
474 as a signal target. */
475 void operator()(float value, float dev=0.0f);
476 ///< Same as operator() with \a min == \a avg == \a max
477 /**< Provided so a Statistics instance can be directly used
478 as a signal target. */
479 void operator()(StatisticsData const & data);
480 ///< Same as operator(), but imports statistics data from a StatisticsData object
481 /**< Provided so a Statistics instance can be directly used
482 as a signal target. */
483 template <class Value>
484 void operator()(unsigned n, StatisticAccumulator<Value> & sa);
485 ///< Same as operator() gathers values from StatisticAccumulator
486 /**< Provided so a Statistics instance can be directly used
487 as a signal target. Caution: Clears values in
488 StatisticAccumulator afterwards
489 \param[in] n number of time-slices
490 \param[in] sa StatisticAccumulator*/
492 StatisticsBase::OutputProxy<Statistics> output(unsigned n = 1u);
494 void consoleList(std::ostream & os) const;
495 void consoleCollect(std::vector<unsigned> & ranks);
496 boost::shared_ptr<senf::console::DirectoryNode> consoleOutput(
497 std::vector<unsigned> & ranks, unsigned window);
500 Statistics & v_base();
501 std::string v_path() const;
504 /** \brief Accumulated statistics collector
506 This class collects accumulated statistics. It is automatically created by
507 senf::Statistics::collect()
509 \see senf::Statistics
511 class Collector : public StatisticsBase
514 virtual unsigned rank() const;
516 StatisticsBase::OutputProxy<Collector> output(unsigned n = 1u);
519 Collector(StatisticsBase * owner, unsigned rank);
520 void enter(unsigned n, float min, float avg, float max, float dev);
521 Statistics & v_base();
522 std::string v_path() const;
530 StatisticsBase * owner_;
532 friend class StatisticsBase;
537 //-/////////////////////////////////////////////////////////////////////////////////////////////////
538 #include "Statistics.cci"
539 //#include "Statistics.ct"
540 #include "Statistics.cti"
547 // comment-column: 40
548 // c-file-style: "senf"
549 // indent-tabs-mode: nil
550 // ispell-local-dictionary: "american"
551 // compile-command: "scons -u test"