# HG changeset patch # User tomwalters # Date 1265977883 0 # Node ID 582cbe817f2c51a9fd790ce762a6f172c6aa6a3a - Initial add of support code and modules. Not everything is working yet. diff -r 000000000000 -r 582cbe817f2c SConstruct --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/SConstruct Fri Feb 12 12:31:23 2010 +0000 @@ -0,0 +1,125 @@ +# Copyright 2006-2010, Willem van Engen, Thomas Walters +# +# AIM-C: A C++ implementation of the Auditory Image Model +# http://www.acousticscale.org/AIMC +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +## @author Thomas Walters +# @author Willem van Engen +# @date created 2010/02/02 +# @version \$Id$ + +"""@package SConstruct +SConstruct file for the aimc project + +""" + +import os +import shutil + +# Define the command-line options for running scons +options = Variables() +options.Add(BoolVariable('mingw', + 'Compile on Windows using mingw rather than msvc', + False)) + +# Environment variables +env = Environment(options = options, ENV = os.environ) +if env['mingw']: + # SCons Defaults to MSVC if installed on Windows. + env = Environment(options = opts, ENV = os.environ, tools = ['mingw']) + +# Platform +build_platform = env['PLATFORM'] +target_platform = build_platform + +# Build products location and executable name +build_dir = os.path.join('build', target_platform + '-release') +target_executable = 'aimc' + +# Create build products directory if necessary +if not os.path.exists(build_dir): + os.makedirs(build_dir) +env.SConsignFile(os.path.join(build_dir,'.sconsign')) + +# Set any platform-specific environment variables and options +if target_platform == 'win32': + env.AppendUnique(CPPDEFINES = ['_WINDOWS', 'WIN32', + 'WINVER=0x0400', '_CONSOLE']) +elif target_platform == 'darwin': + env.AppendUnique(CPPDEFINES = ['_MACOSX']) + +# Compiler selection based on platform +# compiler can be one of: gcc msvc +compiler = 'gcc' +if (build_platform == 'win32' + and target_platform == 'win32' + and not env['mingw']): + compiler = 'msvc' + +# Compiler-specific options: +# Microsoft visual studio +if compiler == 'msvc': + env.AppendUnique(CPPFLAGS = ['/arch:SSE2', '/nologo', '/W3', '/EHsc']) + env.AppendUnique(CPPDEFINES = ['_CRT_SECURE_NO_DEPRECATE', + '_RINT_REQUIRED']) + env.AppendUnique(CPPFLAGS = ['/Ox']) + env.AppendUnique(CPPDEFINES = ['NDEBUG', '_ATL_MIN_CRT']) + +# GNU compiler collection +elif compiler == 'gcc': + env['STRIP'] = 'strip' + env.AppendUnique(CPPFLAGS = ['-Wall']) + env.AppendUnique(CPPFLAGS = ['-O3', '-fomit-frame-pointer']) + if env['mingw']: + if not env['PLATFORM'] == 'win32': + print('Cross-compilation for Windows is not supported') + Exit(1) +else: + print('Unsupported compiler: ' + compiler) + Exit(1) + +# Sources common to every version +common_sources = ['Support/Common.cc', + 'Support/Parameters.cc', + 'Support/Module.cc', + 'Modules/BMM/ModulePZFC.cc', + 'Modules/NAP/ModuleHCL.cc', + 'Modules/SAI/ModuleSAI.cc', + 'Modules/Features/ModuleGaussians.cc'] + +if not target_platform == 'win32': + # On windows, utf support is builtin for SimpleIni + # bit not on other platforms + common_sources += ['Support/ConvertUTF.c'] + +# Choose file which contains main() +sources = common_sources + ['Main/aimc.cpp'] + +# Place the build products in the corect place +env.BuildDir('#' + build_dir, '#', duplicate = 0) + +# Look for the sources in the correct place +env.Append(CPPPATH = '#src') + +# Dependencies +deplibs = '' +env.AppendUnique(LIBS = deplibs) + +# Set up the builder to build the program +program = env.Program(target = os.path.join(build_dir, target_executable), + source = map(lambda x: '#' + build_dir + '/src/' + x, + sources)) +env.Default(program) diff -r 000000000000 -r 582cbe817f2c doc/dox-mainpage.dox --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/doc/dox-mainpage.dox Fri Feb 12 12:31:23 2010 +0000 @@ -0,0 +1,49 @@ +/*! +\mainpage AIM-C developer documentation + +AIM-C is a real-time version of the the auditory image model (AIM) +developed at the +Centre for the Neural Basis of Hearing. + +AIM-C uses a block-based processing scheme. A mono input signal is +split into short segments, which are sequentially 'pushed' through the +processing pipeline. The pipeline consists of a number of modules in a +tree structure. + +The basic unit of data in AIM-C is the signal bank, and the basic +processing unit is the module. A signal bank represents a short +segment, or 'frame', of an audio signal with multiple +channels. Modules generally take a frame (represented as a signal +bank) as input, and generate zero, one, or more signal bank frames as +output. Each module is linked to a set of 'target' modules, to which +they 'push' the frames which they generate. Each of these modules +performs processing on the output of the previous module, and in turn, +push the output that they produce to their targets. + +This feed-forward, tree-like system allows modules to retain the internal +state required for processing + + + + +You can browse the classes using the browser at the left, or +select a page from the contents below. + + + + + +Coding style + +For the most part, AIM-C now follows the +Google C++ style guide. The original (2006-2009) AIM-C codebase +used a different style, and so you may find some files which still +follow the older conventions. New code should adhere to the Google style +guide. + + + + + +*/ diff -r 000000000000 -r 582cbe817f2c lint/cpplint.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/lint/cpplint.py Fri Feb 12 12:31:23 2010 +0000 @@ -0,0 +1,3046 @@ +#!/usr/bin/python +# +# Copyright (c) 2009 Google Inc. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# Here are some issues that I've had people identify in my code during reviews, +# that I think are possible to flag automatically in a lint tool. If these were +# caught by lint, it would save time both for myself and that of my reviewers. +# Most likely, some of these are beyond the scope of the current lint framework, +# but I think it is valuable to retain these wish-list items even if they cannot +# be immediately implemented. +# +# Suggestions +# ----------- +# - Check for no 'explicit' for multi-arg ctor +# - Check for boolean assign RHS in parens +# - Check for ctor initializer-list colon position and spacing +# - Check that if there's a ctor, there should be a dtor +# - Check accessors that return non-pointer member variables are +# declared const +# - Check accessors that return non-const pointer member vars are +# *not* declared const +# - Check for using public includes for testing +# - Check for spaces between brackets in one-line inline method +# - Check for no assert() +# - Check for spaces surrounding operators +# - Check for 0 in pointer context (should be NULL) +# - Check for 0 in char context (should be '\0') +# - Check for camel-case method name conventions for methods +# that are not simple inline getters and setters +# - Check that base classes have virtual destructors +# put " // namespace" after } that closes a namespace, with +# namespace's name after 'namespace' if it is named. +# - Do not indent namespace contents +# - Avoid inlining non-trivial constructors in header files +# include base/basictypes.h if DISALLOW_EVIL_CONSTRUCTORS is used +# - Check for old-school (void) cast for call-sites of functions +# ignored return value +# - Check gUnit usage of anonymous namespace +# - Check for class declaration order (typedefs, consts, enums, +# ctor(s?), dtor, friend declarations, methods, member vars) +# + +"""Does google-lint on c++ files. + +The goal of this script is to identify places in the code that *may* +be in non-compliance with google style. It does not attempt to fix +up these problems -- the point is to educate. It does also not +attempt to find all problems, or to ensure that everything it does +find is legitimately a problem. + +In particular, we can get very confused by /* and // inside strings! +We do a small hack, which is to ignore //'s with "'s after them on the +same line, but it is far from perfect (in either direction). +""" + +import codecs +import getopt +import math # for log +import os +import re +import sre_compile +import string +import sys +import unicodedata + + +_USAGE = """ +Syntax: cpplint.py [--verbose=#] [--output=vs7] [--filter=-x,+y,...] + [--counting=total|toplevel|detailed] + [file] ... + + The style guidelines this tries to follow are those in + http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml + + Every problem is given a confidence score from 1-5, with 5 meaning we are + certain of the problem, and 1 meaning it could be a legitimate construct. + This will miss some errors, and is not a substitute for a code review. + + To prevent specific lines from being linted, add a '// NOLINT' comment to the + end of the line. + + The files passed in will be linted; at least one file must be provided. + Linted extensions are .cc, .cpp, and .h. Other file types will be ignored. + + Flags: + + output=vs7 + By default, the output is formatted to ease emacs parsing. Visual Studio + compatible output (vs7) may also be used. Other formats are unsupported. + + verbose=# + Specify a number 0-5 to restrict errors to certain verbosity levels. + + filter=-x,+y,... + Specify a comma-separated list of category-filters to apply: only + error messages whose category names pass the filters will be printed. + (Category names are printed with the message and look like + "[whitespace/indent]".) Filters are evaluated left to right. + "-FOO" and "FOO" means "do not print categories that start with FOO". + "+FOO" means "do print categories that start with FOO". + + Examples: --filter=-whitespace,+whitespace/braces + --filter=whitespace,runtime/printf,+runtime/printf_format + --filter=-,+build/include_what_you_use + + To see a list of all the categories used in cpplint, pass no arg: + --filter= + + counting=total|toplevel|detailed + The total number of errors found is always printed. If + 'toplevel' is provided, then the count of errors in each of + the top-level categories like 'build' and 'whitespace' will + also be printed. If 'detailed' is provided, then a count + is provided for each category like 'build/class'. +""" + +# We categorize each error message we print. Here are the categories. +# We want an explicit list so we can list them all in cpplint --filter=. +# If you add a new error message with a new category, add it to the list +# here! cpplint_unittest.py should tell you if you forget to do this. +# \ used for clearer layout -- pylint: disable-msg=C6013 +_ERROR_CATEGORIES = '''\ + build/class + build/deprecated + build/endif_comment + build/forward_decl + build/header_guard + build/include + build/include_alpha + build/include_order + build/include_what_you_use + build/namespaces + build/printf_format + build/storage_class + legal/copyright + readability/braces + readability/casting + readability/check + readability/constructors + readability/fn_size + readability/function + readability/multiline_comment + readability/multiline_string + readability/streams + readability/todo + readability/utf8 + runtime/arrays + runtime/casting + runtime/explicit + runtime/int + runtime/init + runtime/invalid_increment + runtime/member_string_references + runtime/memset + runtime/operator + runtime/printf + runtime/printf_format + runtime/references + runtime/rtti + runtime/sizeof + runtime/string + runtime/threadsafe_fn + runtime/virtual + whitespace/blank_line + whitespace/braces + whitespace/comma + whitespace/comments + whitespace/end_of_line + whitespace/ending_newline + whitespace/indent + whitespace/labels + whitespace/line_length + whitespace/newline + whitespace/operators + whitespace/parens + whitespace/semicolon + whitespace/tab + whitespace/todo +''' + +# The default state of the category filter. This is overrided by the --filter= +# flag. By default all errors are on, so only add here categories that should be +# off by default (i.e., categories that must be enabled by the --filter= flags). +# All entries here should start with a '-' or '+', as in the --filter= flag. +_DEFAULT_FILTERS = [ '-build/include_alpha' ] + +# We used to check for high-bit characters, but after much discussion we +# decided those were OK, as long as they were in UTF-8 and didn't represent +# hard-coded international strings, which belong in a seperate i18n file. + +# Headers that we consider STL headers. +_STL_HEADERS = frozenset([ + 'algobase.h', 'algorithm', 'alloc.h', 'bitset', 'deque', 'exception', + 'function.h', 'functional', 'hash_map', 'hash_map.h', 'hash_set', + 'hash_set.h', 'iterator', 'list', 'list.h', 'map', 'memory', 'pair.h', + 'pthread_alloc', 'queue', 'set', 'set.h', 'sstream', 'stack', + 'stl_alloc.h', 'stl_relops.h', 'type_traits.h', + 'utility', 'vector', 'vector.h', + ]) + + +# Non-STL C++ system headers. +_CPP_HEADERS = frozenset([ + 'algo.h', 'builtinbuf.h', 'bvector.h', 'cassert', 'cctype', + 'cerrno', 'cfloat', 'ciso646', 'climits', 'clocale', 'cmath', + 'complex', 'complex.h', 'csetjmp', 'csignal', 'cstdarg', 'cstddef', + 'cstdio', 'cstdlib', 'cstring', 'ctime', 'cwchar', 'cwctype', + 'defalloc.h', 'deque.h', 'editbuf.h', 'exception', 'fstream', + 'fstream.h', 'hashtable.h', 'heap.h', 'indstream.h', 'iomanip', + 'iomanip.h', 'ios', 'iosfwd', 'iostream', 'iostream.h', 'istream.h', + 'iterator.h', 'limits', 'map.h', 'multimap.h', 'multiset.h', + 'numeric', 'ostream.h', 'parsestream.h', 'pfstream.h', 'PlotFile.h', + 'procbuf.h', 'pthread_alloc.h', 'rope', 'rope.h', 'ropeimpl.h', + 'SFile.h', 'slist', 'slist.h', 'stack.h', 'stdexcept', + 'stdiostream.h', 'streambuf.h', 'stream.h', 'strfile.h', 'string', + 'strstream', 'strstream.h', 'tempbuf.h', 'tree.h', 'typeinfo', 'valarray', + ]) + + +# Assertion macros. These are defined in base/logging.h and +# testing/base/gunit.h. Note that the _M versions need to come first +# for substring matching to work. +_CHECK_MACROS = [ + 'DCHECK', 'CHECK', + 'EXPECT_TRUE_M', 'EXPECT_TRUE', + 'ASSERT_TRUE_M', 'ASSERT_TRUE', + 'EXPECT_FALSE_M', 'EXPECT_FALSE', + 'ASSERT_FALSE_M', 'ASSERT_FALSE', + ] + +# Replacement macros for CHECK/DCHECK/EXPECT_TRUE/EXPECT_FALSE +_CHECK_REPLACEMENT = dict([(m, {}) for m in _CHECK_MACROS]) + +for op, replacement in [('==', 'EQ'), ('!=', 'NE'), + ('>=', 'GE'), ('>', 'GT'), + ('<=', 'LE'), ('<', 'LT')]: + _CHECK_REPLACEMENT['DCHECK'][op] = 'DCHECK_%s' % replacement + _CHECK_REPLACEMENT['CHECK'][op] = 'CHECK_%s' % replacement + _CHECK_REPLACEMENT['EXPECT_TRUE'][op] = 'EXPECT_%s' % replacement + _CHECK_REPLACEMENT['ASSERT_TRUE'][op] = 'ASSERT_%s' % replacement + _CHECK_REPLACEMENT['EXPECT_TRUE_M'][op] = 'EXPECT_%s_M' % replacement + _CHECK_REPLACEMENT['ASSERT_TRUE_M'][op] = 'ASSERT_%s_M' % replacement + +for op, inv_replacement in [('==', 'NE'), ('!=', 'EQ'), + ('>=', 'LT'), ('>', 'LE'), + ('<=', 'GT'), ('<', 'GE')]: + _CHECK_REPLACEMENT['EXPECT_FALSE'][op] = 'EXPECT_%s' % inv_replacement + _CHECK_REPLACEMENT['ASSERT_FALSE'][op] = 'ASSERT_%s' % inv_replacement + _CHECK_REPLACEMENT['EXPECT_FALSE_M'][op] = 'EXPECT_%s_M' % inv_replacement + _CHECK_REPLACEMENT['ASSERT_FALSE_M'][op] = 'ASSERT_%s_M' % inv_replacement + + +# These constants define types of headers for use with +# _IncludeState.CheckNextIncludeOrder(). +_C_SYS_HEADER = 1 +_CPP_SYS_HEADER = 2 +_LIKELY_MY_HEADER = 3 +_POSSIBLE_MY_HEADER = 4 +_OTHER_HEADER = 5 + + +_regexp_compile_cache = {} + + +def Match(pattern, s): + """Matches the string with the pattern, caching the compiled regexp.""" + # The regexp compilation caching is inlined in both Match and Search for + # performance reasons; factoring it out into a separate function turns out + # to be noticeably expensive. + if not pattern in _regexp_compile_cache: + _regexp_compile_cache[pattern] = sre_compile.compile(pattern) + return _regexp_compile_cache[pattern].match(s) + + +def Search(pattern, s): + """Searches the string for the pattern, caching the compiled regexp.""" + if not pattern in _regexp_compile_cache: + _regexp_compile_cache[pattern] = sre_compile.compile(pattern) + return _regexp_compile_cache[pattern].search(s) + + +class _IncludeState(dict): + """Tracks line numbers for includes, and the order in which includes appear. + + As a dict, an _IncludeState object serves as a mapping between include + filename and line number on which that file was included. + + Call CheckNextIncludeOrder() once for each header in the file, passing + in the type constants defined above. Calls in an illegal order will + raise an _IncludeError with an appropriate error message. + + """ + # self._section will move monotonically through this set. If it ever + # needs to move backwards, CheckNextIncludeOrder will raise an error. + _INITIAL_SECTION = 0 + _MY_H_SECTION = 1 + _C_SECTION = 2 + _CPP_SECTION = 3 + _OTHER_H_SECTION = 4 + + _TYPE_NAMES = { + _C_SYS_HEADER: 'C system header', + _CPP_SYS_HEADER: 'C++ system header', + _LIKELY_MY_HEADER: 'header this file implements', + _POSSIBLE_MY_HEADER: 'header this file may implement', + _OTHER_HEADER: 'other header', + } + _SECTION_NAMES = { + _INITIAL_SECTION: "... nothing. (This can't be an error.)", + _MY_H_SECTION: 'a header this file implements', + _C_SECTION: 'C system header', + _CPP_SECTION: 'C++ system header', + _OTHER_H_SECTION: 'other header', + } + + def __init__(self): + dict.__init__(self) + # The name of the current section. + self._section = self._INITIAL_SECTION + # The path of last found header. + self._last_header = '' + + def CanonicalizeAlphabeticalOrder(self, header_path): + """Returns a path canonicalized for alphabetical comparisson. + + - replaces "-" with "_" so they both cmp the same. + - removes '-inl' since we don't require them to be after the main header. + - lowercase everything, just in case. + + Args: + header_path: Path to be canonicalized. + + Returns: + Canonicalized path. + """ + return header_path.replace('-inl.h', '.h').replace('-', '_').lower() + + def IsInAlphabeticalOrder(self, header_path): + """Check if a header is in alphabetical order with the previous header. + + Args: + header_path: Header to be checked. + + Returns: + Returns true if the header is in alphabetical order. + """ + canonical_header = self.CanonicalizeAlphabeticalOrder(header_path) + if self._last_header > canonical_header: + return False + self._last_header = canonical_header + return True + + def CheckNextIncludeOrder(self, header_type): + """Returns a non-empty error message if the next header is out of order. + + This function also updates the internal state to be ready to check + the next include. + + Args: + header_type: One of the _XXX_HEADER constants defined above. + + Returns: + The empty string if the header is in the right order, or an + error message describing what's wrong. + + """ + error_message = ('Found %s after %s' % + (self._TYPE_NAMES[header_type], + self._SECTION_NAMES[self._section])) + + last_section = self._section + + if header_type == _C_SYS_HEADER: + if self._section <= self._C_SECTION: + self._section = self._C_SECTION + else: + self._last_header = '' + return error_message + elif header_type == _CPP_SYS_HEADER: + if self._section <= self._CPP_SECTION: + self._section = self._CPP_SECTION + else: + self._last_header = '' + return error_message + elif header_type == _LIKELY_MY_HEADER: + if self._section <= self._MY_H_SECTION: + self._section = self._MY_H_SECTION + else: + self._section = self._OTHER_H_SECTION + elif header_type == _POSSIBLE_MY_HEADER: + if self._section <= self._MY_H_SECTION: + self._section = self._MY_H_SECTION + else: + # This will always be the fallback because we're not sure + # enough that the header is associated with this file. + self._section = self._OTHER_H_SECTION + else: + assert header_type == _OTHER_HEADER + self._section = self._OTHER_H_SECTION + + if last_section != self._section: + self._last_header = '' + + return '' + + +class _CppLintState(object): + """Maintains module-wide state..""" + + def __init__(self): + self.verbose_level = 1 # global setting. + self.error_count = 0 # global count of reported errors + # filters to apply when emitting error messages + self.filters = _DEFAULT_FILTERS[:] + self.counting = 'total' # In what way are we counting errors? + self.errors_by_category = {} # string to int dict storing error counts + + # output format: + # "emacs" - format that emacs can parse (default) + # "vs7" - format that Microsoft Visual Studio 7 can parse + self.output_format = 'emacs' + + def SetOutputFormat(self, output_format): + """Sets the output format for errors.""" + self.output_format = output_format + + def SetVerboseLevel(self, level): + """Sets the module's verbosity, and returns the previous setting.""" + last_verbose_level = self.verbose_level + self.verbose_level = level + return last_verbose_level + + def SetCountingStyle(self, counting_style): + """Sets the module's counting options.""" + self.counting = counting_style + + def SetFilters(self, filters): + """Sets the error-message filters. + + These filters are applied when deciding whether to emit a given + error message. + + Args: + filters: A string of comma-separated filters (eg "+whitespace/indent"). + Each filter should start with + or -; else we die. + + Raises: + ValueError: The comma-separated filters did not all start with '+' or '-'. + E.g. "-,+whitespace,-whitespace/indent,whitespace/badfilter" + """ + # Default filters always have less priority than the flag ones. + self.filters = _DEFAULT_FILTERS[:] + for filt in filters.split(','): + clean_filt = filt.strip() + if clean_filt: + self.filters.append(clean_filt) + for filt in self.filters: + if not (filt.startswith('+') or filt.startswith('-')): + raise ValueError('Every filter in --filters must start with + or -' + ' (%s does not)' % filt) + + def ResetErrorCounts(self): + """Sets the module's error statistic back to zero.""" + self.error_count = 0 + self.errors_by_category = {} + + def IncrementErrorCount(self, category): + """Bumps the module's error statistic.""" + self.error_count += 1 + if self.counting in ('toplevel', 'detailed'): + if self.counting != 'detailed': + category = category.split('/')[0] + if category not in self.errors_by_category: + self.errors_by_category[category] = 0 + self.errors_by_category[category] += 1 + + def PrintErrorCounts(self): + """Print a summary of errors by category, and the total.""" + for category, count in self.errors_by_category.iteritems(): + sys.stderr.write('Category \'%s\' errors found: %d\n' % + (category, count)) + sys.stderr.write('Total errors found: %d\n' % self.error_count) + +_cpplint_state = _CppLintState() + + +def _OutputFormat(): + """Gets the module's output format.""" + return _cpplint_state.output_format + + +def _SetOutputFormat(output_format): + """Sets the module's output format.""" + _cpplint_state.SetOutputFormat(output_format) + + +def _VerboseLevel(): + """Returns the module's verbosity setting.""" + return _cpplint_state.verbose_level + + +def _SetVerboseLevel(level): + """Sets the module's verbosity, and returns the previous setting.""" + return _cpplint_state.SetVerboseLevel(level) + + +def _SetCountingStyle(level): + """Sets the module's counting options.""" + _cpplint_state.SetCountingStyle(level) + + +def _Filters(): + """Returns the module's list of output filters, as a list.""" + return _cpplint_state.filters + + +def _SetFilters(filters): + """Sets the module's error-message filters. + + These filters are applied when deciding whether to emit a given + error message. + + Args: + filters: A string of comma-separated filters (eg "whitespace/indent"). + Each filter should start with + or -; else we die. + """ + _cpplint_state.SetFilters(filters) + + +class _FunctionState(object): + """Tracks current function name and the number of lines in its body.""" + + _NORMAL_TRIGGER = 250 # for --v=0, 500 for --v=1, etc. + _TEST_TRIGGER = 400 # about 50% more than _NORMAL_TRIGGER. + + def __init__(self): + self.in_a_function = False + self.lines_in_function = 0 + self.current_function = '' + + def Begin(self, function_name): + """Start analyzing function body. + + Args: + function_name: The name of the function being tracked. + """ + self.in_a_function = True + self.lines_in_function = 0 + self.current_function = function_name + + def Count(self): + """Count line in current function body.""" + if self.in_a_function: + self.lines_in_function += 1 + + def Check(self, error, filename, linenum): + """Report if too many lines in function body. + + Args: + error: The function to call with any errors found. + filename: The name of the current file. + linenum: The number of the line to check. + """ + if Match(r'T(EST|est)', self.current_function): + base_trigger = self._TEST_TRIGGER + else: + base_trigger = self._NORMAL_TRIGGER + trigger = base_trigger * 2**_VerboseLevel() + + if self.lines_in_function > trigger: + error_level = int(math.log(self.lines_in_function / base_trigger, 2)) + # 50 => 0, 100 => 1, 200 => 2, 400 => 3, 800 => 4, 1600 => 5, ... + if error_level > 5: + error_level = 5 + error(filename, linenum, 'readability/fn_size', error_level, + 'Small and focused functions are preferred:' + ' %s has %d non-comment lines' + ' (error triggered by exceeding %d lines).' % ( + self.current_function, self.lines_in_function, trigger)) + + def End(self): + """Stop analizing function body.""" + self.in_a_function = False + + +class _IncludeError(Exception): + """Indicates a problem with the include order in a file.""" + pass + + +class FileInfo: + """Provides utility functions for filenames. + + FileInfo provides easy access to the components of a file's path + relative to the project root. + """ + + def __init__(self, filename): + self._filename = filename + + def FullName(self): + """Make Windows paths like Unix.""" + return os.path.abspath(self._filename).replace('\\', '/') + + def RepositoryName(self): + """FullName after removing the local path to the repository. + + If we have a real absolute path name here we can try to do something smart: + detecting the root of the checkout and truncating /path/to/checkout from + the name so that we get header guards that don't include things like + "C:\Documents and Settings\..." or "/home/username/..." in them and thus + people on different computers who have checked the source out to different + locations won't see bogus errors. + """ + fullname = self.FullName() + + if os.path.exists(fullname): + project_dir = os.path.dirname(fullname) + + if os.path.exists(os.path.join(project_dir, ".svn")): + # If there's a .svn file in the current directory, we recursively look + # up the directory tree for the top of the SVN checkout + root_dir = project_dir + one_up_dir = os.path.dirname(root_dir) + while os.path.exists(os.path.join(one_up_dir, ".svn")): + root_dir = os.path.dirname(root_dir) + one_up_dir = os.path.dirname(one_up_dir) + + prefix = os.path.commonprefix([root_dir, project_dir]) + return fullname[len(prefix) + 1:] + + # Not SVN? Try to find a git or hg top level directory by searching up + # from the current path. + root_dir = os.path.dirname(fullname) + while (root_dir != os.path.dirname(root_dir) and + not os.path.exists(os.path.join(root_dir, ".git")) and + not os.path.exists(os.path.join(root_dir, ".hg"))): + root_dir = os.path.dirname(root_dir) + if (os.path.exists(os.path.join(root_dir, ".git")) or + os.path.exists(os.path.join(root_dir, ".hg"))): + prefix = os.path.commonprefix([root_dir, project_dir]) + return fullname[len(prefix) + 1:] + + # Don't know what to do; header guard warnings may be wrong... + return fullname + + def Split(self): + """Splits the file into the directory, basename, and extension. + + For 'chrome/browser/browser.cc', Split() would + return ('chrome/browser', 'browser', '.cc') + + Returns: + A tuple of (directory, basename, extension). + """ + + googlename = self.RepositoryName() + project, rest = os.path.split(googlename) + return (project,) + os.path.splitext(rest) + + def BaseName(self): + """File base name - text after the final slash, before the final period.""" + return self.Split()[1] + + def Extension(self): + """File extension - text following the final period.""" + return self.Split()[2] + + def NoExtension(self): + """File has no source file extension.""" + return '/'.join(self.Split()[0:2]) + + def IsSource(self): + """File has a source file extension.""" + return self.Extension()[1:] in ('c', 'cc', 'cpp', 'cxx') + + +def _ShouldPrintError(category, confidence): + """Returns true iff confidence >= verbose, and category passes filter.""" + # There are two ways we might decide not to print an error message: + # the verbosity level isn't high enough, or the filters filter it out. + if confidence < _cpplint_state.verbose_level: + return False + + is_filtered = False + for one_filter in _Filters(): + if one_filter.startswith('-'): + if category.startswith(one_filter[1:]): + is_filtered = True + elif one_filter.startswith('+'): + if category.startswith(one_filter[1:]): + is_filtered = False + else: + assert False # should have been checked for in SetFilter. + if is_filtered: + return False + + return True + + +def Error(filename, linenum, category, confidence, message): + """Logs the fact we've found a lint error. + + We log where the error was found, and also our confidence in the error, + that is, how certain we are this is a legitimate style regression, and + not a misidentification or a use that's sometimes justified. + + Args: + filename: The name of the file containing the error. + linenum: The number of the line containing the error. + category: A string used to describe the "category" this bug + falls under: "whitespace", say, or "runtime". Categories + may have a hierarchy separated by slashes: "whitespace/indent". + confidence: A number from 1-5 representing a confidence score for + the error, with 5 meaning that we are certain of the problem, + and 1 meaning that it could be a legitimate construct. + message: The error message. + """ + # There are two ways we might decide not to print an error message: + # the verbosity level isn't high enough, or the filters filter it out. + if _ShouldPrintError(category, confidence): + _cpplint_state.IncrementErrorCount(category) + if _cpplint_state.output_format == 'vs7': + sys.stderr.write('%s(%s): %s [%s] [%d]\n' % ( + filename, linenum, message, category, confidence)) + else: + sys.stderr.write('%s:%s: %s [%s] [%d]\n' % ( + filename, linenum, message, category, confidence)) + + +# Matches standard C++ escape esequences per 2.13.2.3 of the C++ standard. +_RE_PATTERN_CLEANSE_LINE_ESCAPES = re.compile( + r'\\([abfnrtv?"\\\']|\d+|x[0-9a-fA-F]+)') +# Matches strings. Escape codes should already be removed by ESCAPES. +_RE_PATTERN_CLEANSE_LINE_DOUBLE_QUOTES = re.compile(r'"[^"]*"') +# Matches characters. Escape codes should already be removed by ESCAPES. +_RE_PATTERN_CLEANSE_LINE_SINGLE_QUOTES = re.compile(r"'.'") +# Matches multi-line C++ comments. +# This RE is a little bit more complicated than one might expect, because we +# have to take care of space removals tools so we can handle comments inside +# statements better. +# The current rule is: We only clear spaces from both sides when we're at the +# end of the line. Otherwise, we try to remove spaces from the right side, +# if this doesn't work we try on left side but only if there's a non-character +# on the right. +_RE_PATTERN_CLEANSE_LINE_C_COMMENTS = re.compile( + r"""(\s*/\*.*\*/\s*$| + /\*.*\*/\s+| + \s+/\*.*\*/(?=\W)| + /\*.*\*/)""", re.VERBOSE) + + +def IsCppString(line): + """Does line terminate so, that the next symbol is in string constant. + + This function does not consider single-line nor multi-line comments. + + Args: + line: is a partial line of code starting from the 0..n. + + Returns: + True, if next character appended to 'line' is inside a + string constant. + """ + + line = line.replace(r'\\', 'XX') # after this, \\" does not match to \" + return ((line.count('"') - line.count(r'\"') - line.count("'\"'")) & 1) == 1 + + +def FindNextMultiLineCommentStart(lines, lineix): + """Find the beginning marker for a multiline comment.""" + while lineix < len(lines): + if lines[lineix].strip().startswith('/*'): + # Only return this marker if the comment goes beyond this line + if lines[lineix].strip().find('*/', 2) < 0: + return lineix + lineix += 1 + return len(lines) + + +def FindNextMultiLineCommentEnd(lines, lineix): + """We are inside a comment, find the end marker.""" + while lineix < len(lines): + if lines[lineix].strip().endswith('*/'): + return lineix + lineix += 1 + return len(lines) + + +def RemoveMultiLineCommentsFromRange(lines, begin, end): + """Clears a range of lines for multi-line comments.""" + # Having // dummy comments makes the lines non-empty, so we will not get + # unnecessary blank line warnings later in the code. + for i in range(begin, end): + lines[i] = '// dummy' + + +def RemoveMultiLineComments(filename, lines, error): + """Removes multiline (c-style) comments from lines.""" + lineix = 0 + while lineix < len(lines): + lineix_begin = FindNextMultiLineCommentStart(lines, lineix) + if lineix_begin >= len(lines): + return + lineix_end = FindNextMultiLineCommentEnd(lines, lineix_begin) + if lineix_end >= len(lines): + error(filename, lineix_begin + 1, 'readability/multiline_comment', 5, + 'Could not find end of multi-line comment') + return + RemoveMultiLineCommentsFromRange(lines, lineix_begin, lineix_end + 1) + lineix = lineix_end + 1 + + +def CleanseComments(line): + """Removes //-comments and single-line C-style /* */ comments. + + Args: + line: A line of C++ source. + + Returns: + The line with single-line comments removed. + """ + commentpos = line.find('//') + if commentpos != -1 and not IsCppString(line[:commentpos]): + line = line[:commentpos] + # get rid of /* ... */ + return _RE_PATTERN_CLEANSE_LINE_C_COMMENTS.sub('', line) + + +class CleansedLines(object): + """Holds 3 copies of all lines with different preprocessing applied to them. + + 1) elided member contains lines without strings and comments, + 2) lines member contains lines without comments, and + 3) raw member contains all the lines without processing. + All these three members are of , and of the same length. + """ + + def __init__(self, lines): + self.elided = [] + self.lines = [] + self.raw_lines = lines + self.num_lines = len(lines) + for linenum in range(len(lines)): + self.lines.append(CleanseComments(lines[linenum])) + elided = self._CollapseStrings(lines[linenum]) + self.elided.append(CleanseComments(elided)) + + def NumLines(self): + """Returns the number of lines represented.""" + return self.num_lines + + @staticmethod + def _CollapseStrings(elided): + """Collapses strings and chars on a line to simple "" or '' blocks. + + We nix strings first so we're not fooled by text like '"http://"' + + Args: + elided: The line being processed. + + Returns: + The line with collapsed strings. + """ + if not _RE_PATTERN_INCLUDE.match(elided): + # Remove escaped characters first to make quote/single quote collapsing + # basic. Things that look like escaped characters shouldn't occur + # outside of strings and chars. + elided = _RE_PATTERN_CLEANSE_LINE_ESCAPES.sub('', elided) + elided = _RE_PATTERN_CLEANSE_LINE_SINGLE_QUOTES.sub("''", elided) + elided = _RE_PATTERN_CLEANSE_LINE_DOUBLE_QUOTES.sub('""', elided) + return elided + + +def CloseExpression(clean_lines, linenum, pos): + """If input points to ( or { or [, finds the position that closes it. + + If lines[linenum][pos] points to a '(' or '{' or '[', finds the the + linenum/pos that correspond to the closing of the expression. + + Args: + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + pos: A position on the line. + + Returns: + A tuple (line, linenum, pos) pointer *past* the closing brace, or + (line, len(lines), -1) if we never find a close. Note we ignore + strings and comments when matching; and the line we return is the + 'cleansed' line at linenum. + """ + + line = clean_lines.elided[linenum] + startchar = line[pos] + if startchar not in '({[': + return (line, clean_lines.NumLines(), -1) + if startchar == '(': endchar = ')' + if startchar == '[': endchar = ']' + if startchar == '{': endchar = '}' + + num_open = line.count(startchar) - line.count(endchar) + while linenum < clean_lines.NumLines() and num_open > 0: + linenum += 1 + line = clean_lines.elided[linenum] + num_open += line.count(startchar) - line.count(endchar) + # OK, now find the endchar that actually got us back to even + endpos = len(line) + while num_open >= 0: + endpos = line.rfind(')', 0, endpos) + num_open -= 1 # chopped off another ) + return (line, linenum, endpos + 1) + + +def CheckForCopyright(filename, lines, error): + """Logs an error if no Copyright message appears at the top of the file.""" + + # We'll say it should occur by line 10. Don't forget there's a + # dummy line at the front. + for line in xrange(1, min(len(lines), 11)): + if re.search(r'Copyright', lines[line], re.I): break + else: # means no copyright line was found + error(filename, 0, 'legal/copyright', 5, + 'No copyright message found. ' + 'You should have a line: "Copyright [year] "') + + +def GetHeaderGuardCPPVariable(filename): + """Returns the CPP variable that should be used as a header guard. + + Args: + filename: The name of a C++ header file. + + Returns: + The CPP variable that should be used as a header guard in the + named file. + + """ + + fileinfo = FileInfo(filename) + return re.sub(r'[-./\s]', '_', fileinfo.RepositoryName()).upper() + '_' + + +def CheckForHeaderGuard(filename, lines, error): + """Checks that the file contains a header guard. + + Logs an error if no #ifndef header guard is present. For other + headers, checks that the full pathname is used. + + Args: + filename: The name of the C++ header file. + lines: An array of strings, each representing a line of the file. + error: The function to call with any errors found. + """ + + cppvar = GetHeaderGuardCPPVariable(filename) + + ifndef = None + ifndef_linenum = 0 + define = None + endif = None + endif_linenum = 0 + for linenum, line in enumerate(lines): + linesplit = line.split() + if len(linesplit) >= 2: + # find the first occurrence of #ifndef and #define, save arg + if not ifndef and linesplit[0] == '#ifndef': + # set ifndef to the header guard presented on the #ifndef line. + ifndef = linesplit[1] + ifndef_linenum = linenum + if not define and linesplit[0] == '#define': + define = linesplit[1] + # find the last occurrence of #endif, save entire line + if line.startswith('#endif'): + endif = line + endif_linenum = linenum + + if not ifndef or not define or ifndef != define: + error(filename, 0, 'build/header_guard', 5, + 'No #ifndef header guard found, suggested CPP variable is: %s' % + cppvar) + return + + # The guard should be PATH_FILE_H_, but we also allow PATH_FILE_H__ + # for backward compatibility. + if ifndef != cppvar and not Search(r'\bNOLINT\b', lines[ifndef_linenum]): + error_level = 0 + if ifndef != cppvar + '_': + error_level = 5 + + error(filename, ifndef_linenum, 'build/header_guard', error_level, + '#ifndef header guard has wrong style, please use: %s' % cppvar) + + if (endif != ('#endif // %s' % cppvar) and + not Search(r'\bNOLINT\b', lines[endif_linenum])): + error_level = 0 + if endif != ('#endif // %s' % (cppvar + '_')): + error_level = 5 + + error(filename, endif_linenum, 'build/header_guard', error_level, + '#endif line should be "#endif // %s"' % cppvar) + + +def CheckForUnicodeReplacementCharacters(filename, lines, error): + """Logs an error for each line containing Unicode replacement characters. + + These indicate that either the file contained invalid UTF-8 (likely) + or Unicode replacement characters (which it shouldn't). Note that + it's possible for this to throw off line numbering if the invalid + UTF-8 occurred adjacent to a newline. + + Args: + filename: The name of the current file. + lines: An array of strings, each representing a line of the file. + error: The function to call with any errors found. + """ + for linenum, line in enumerate(lines): + if u'\ufffd' in line: + error(filename, linenum, 'readability/utf8', 5, + 'Line contains invalid UTF-8 (or Unicode replacement character).') + + +def CheckForNewlineAtEOF(filename, lines, error): + """Logs an error if there is no newline char at the end of the file. + + Args: + filename: The name of the current file. + lines: An array of strings, each representing a line of the file. + error: The function to call with any errors found. + """ + + # The array lines() was created by adding two newlines to the + # original file (go figure), then splitting on \n. + # To verify that the file ends in \n, we just have to make sure the + # last-but-two element of lines() exists and is empty. + if len(lines) < 3 or lines[-2]: + error(filename, len(lines) - 2, 'whitespace/ending_newline', 5, + 'Could not find a newline character at the end of the file.') + + +def CheckForMultilineCommentsAndStrings(filename, clean_lines, linenum, error): + """Logs an error if we see /* ... */ or "..." that extend past one line. + + /* ... */ comments are legit inside macros, for one line. + Otherwise, we prefer // comments, so it's ok to warn about the + other. Likewise, it's ok for strings to extend across multiple + lines, as long as a line continuation character (backslash) + terminates each line. Although not currently prohibited by the C++ + style guide, it's ugly and unnecessary. We don't do well with either + in this lint program, so we warn about both. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + + # Remove all \\ (escaped backslashes) from the line. They are OK, and the + # second (escaped) slash may trigger later \" detection erroneously. + line = line.replace('\\\\', '') + + if line.count('/*') > line.count('*/'): + error(filename, linenum, 'readability/multiline_comment', 5, + 'Complex multi-line /*...*/-style comment found. ' + 'Lint may give bogus warnings. ' + 'Consider replacing these with //-style comments, ' + 'with #if 0...#endif, ' + 'or with more clearly structured multi-line comments.') + + if (line.count('"') - line.count('\\"')) % 2: + error(filename, linenum, 'readability/multiline_string', 5, + 'Multi-line string ("...") found. This lint script doesn\'t ' + 'do well with such strings, and may give bogus warnings. They\'re ' + 'ugly and unnecessary, and you should use concatenation instead".') + + +threading_list = ( + ('asctime(', 'asctime_r('), + ('ctime(', 'ctime_r('), + ('getgrgid(', 'getgrgid_r('), + ('getgrnam(', 'getgrnam_r('), + ('getlogin(', 'getlogin_r('), + ('getpwnam(', 'getpwnam_r('), + ('getpwuid(', 'getpwuid_r('), + ('gmtime(', 'gmtime_r('), + ('localtime(', 'localtime_r('), + ('rand(', 'rand_r('), + ('readdir(', 'readdir_r('), + ('strtok(', 'strtok_r('), + ('ttyname(', 'ttyname_r('), + ) + + +def CheckPosixThreading(filename, clean_lines, linenum, error): + """Checks for calls to thread-unsafe functions. + + Much code has been originally written without consideration of + multi-threading. Also, engineers are relying on their old experience; + they have learned posix before threading extensions were added. These + tests guide the engineers to use thread-safe functions (when using + posix directly). + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + for single_thread_function, multithread_safe_function in threading_list: + ix = line.find(single_thread_function) + # Comparisons made explicit for clarity -- pylint: disable-msg=C6403 + if ix >= 0 and (ix == 0 or (not line[ix - 1].isalnum() and + line[ix - 1] not in ('_', '.', '>'))): + error(filename, linenum, 'runtime/threadsafe_fn', 2, + 'Consider using ' + multithread_safe_function + + '...) instead of ' + single_thread_function + + '...) for improved thread safety.') + + +# Matches invalid increment: *count++, which moves pointer instead of +# incrementing a value. +_RE_PATTERN_INVALID_INCREMENT = re.compile( + r'^\s*\*\w+(\+\+|--);') + + +def CheckInvalidIncrement(filename, clean_lines, linenum, error): + """Checks for invalid increment *count++. + + For example following function: + void increment_counter(int* count) { + *count++; + } + is invalid, because it effectively does count++, moving pointer, and should + be replaced with ++*count, (*count)++ or *count += 1. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + line = clean_lines.elided[linenum] + if _RE_PATTERN_INVALID_INCREMENT.match(line): + error(filename, linenum, 'runtime/invalid_increment', 5, + 'Changing pointer instead of value (or unused value of operator*).') + + +class _ClassInfo(object): + """Stores information about a class.""" + + def __init__(self, name, linenum): + self.name = name + self.linenum = linenum + self.seen_open_brace = False + self.is_derived = False + self.virtual_method_linenumber = None + self.has_virtual_destructor = False + self.brace_depth = 0 + + +class _ClassState(object): + """Holds the current state of the parse relating to class declarations. + + It maintains a stack of _ClassInfos representing the parser's guess + as to the current nesting of class declarations. The innermost class + is at the top (back) of the stack. Typically, the stack will either + be empty or have exactly one entry. + """ + + def __init__(self): + self.classinfo_stack = [] + + def CheckFinished(self, filename, error): + """Checks that all classes have been completely parsed. + + Call this when all lines in a file have been processed. + Args: + filename: The name of the current file. + error: The function to call with any errors found. + """ + if self.classinfo_stack: + # Note: This test can result in false positives if #ifdef constructs + # get in the way of brace matching. See the testBuildClass test in + # cpplint_unittest.py for an example of this. + error(filename, self.classinfo_stack[0].linenum, 'build/class', 5, + 'Failed to find complete declaration of class %s' % + self.classinfo_stack[0].name) + + +def CheckForNonStandardConstructs(filename, clean_lines, linenum, + class_state, error): + """Logs an error if we see certain non-ANSI constructs ignored by gcc-2. + + Complain about several constructs which gcc-2 accepts, but which are + not standard C++. Warning about these in lint is one way to ease the + transition to new compilers. + - put storage class first (e.g. "static const" instead of "const static"). + - "%lld" instead of %qd" in printf-type functions. + - "%1$d" is non-standard in printf-type functions. + - "\%" is an undefined character escape sequence. + - text after #endif is not allowed. + - invalid inner-style forward declaration. + - >? and ?= and )\?=?\s*(\w+|[+-]?\d+)(\.\d*)?', + line): + error(filename, linenum, 'build/deprecated', 3, + '>? and ))?' + # r'\s*const\s*' + type_name + '\s*&\s*\w+\s*;' + error(filename, linenum, 'runtime/member_string_references', 2, + 'const string& members are dangerous. It is much better to use ' + 'alternatives, such as pointers or simple constants.') + + # Track class entry and exit, and attempt to find cases within the + # class declaration that don't meet the C++ style + # guidelines. Tracking is very dependent on the code matching Google + # style guidelines, but it seems to perform well enough in testing + # to be a worthwhile addition to the checks. + classinfo_stack = class_state.classinfo_stack + # Look for a class declaration + class_decl_match = Match( + r'\s*(template\s*<[\w\s<>,:]*>\s*)?(class|struct)\s+(\w+(::\w+)*)', line) + if class_decl_match: + classinfo_stack.append(_ClassInfo(class_decl_match.group(3), linenum)) + + # Everything else in this function uses the top of the stack if it's + # not empty. + if not classinfo_stack: + return + + classinfo = classinfo_stack[-1] + + # If the opening brace hasn't been seen look for it and also + # parent class declarations. + if not classinfo.seen_open_brace: + # If the line has a ';' in it, assume it's a forward declaration or + # a single-line class declaration, which we won't process. + if line.find(';') != -1: + classinfo_stack.pop() + return + classinfo.seen_open_brace = (line.find('{') != -1) + # Look for a bare ':' + if Search('(^|[^:]):($|[^:])', line): + classinfo.is_derived = True + if not classinfo.seen_open_brace: + return # Everything else in this function is for after open brace + + # The class may have been declared with namespace or classname qualifiers. + # The constructor and destructor will not have those qualifiers. + base_classname = classinfo.name.split('::')[-1] + + # Look for single-argument constructors that aren't marked explicit. + # Technically a valid construct, but against style. + args = Match(r'(? 1: + error(filename, linenum, 'whitespace/todo', 2, + 'Too many spaces before TODO') + + username = match.group(2) + if not username: + error(filename, linenum, 'readability/todo', 2, + 'Missing username in TODO; it should look like ' + '"// TODO(my_username): Stuff."') + + middle_whitespace = match.group(3) + # Comparisons made explicit for correctness -- pylint: disable-msg=C6403 + if middle_whitespace != ' ' and middle_whitespace != '': + error(filename, linenum, 'whitespace/todo', 2, + 'TODO(my_username) should be followed by a space') + + +def CheckSpacing(filename, clean_lines, linenum, error): + """Checks for the correctness of various spacing issues in the code. + + Things we check for: spaces around operators, spaces after + if/for/while/switch, no spaces around parens in function calls, two + spaces between code and comment, don't start a block with a blank + line, don't end a function with a blank line, don't have too many + blank lines in a row. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + + raw = clean_lines.raw_lines + line = raw[linenum] + + # Before nixing comments, check if the line is blank for no good + # reason. This includes the first line after a block is opened, and + # blank lines at the end of a function (ie, right before a line like '}' + if IsBlankLine(line): + elided = clean_lines.elided + prev_line = elided[linenum - 1] + prevbrace = prev_line.rfind('{') + # TODO(unknown): Don't complain if line before blank line, and line after, + # both start with alnums and are indented the same amount. + # This ignores whitespace at the start of a namespace block + # because those are not usually indented. + if (prevbrace != -1 and prev_line[prevbrace:].find('}') == -1 + and prev_line[:prevbrace].find('namespace') == -1): + # OK, we have a blank line at the start of a code block. Before we + # complain, we check if it is an exception to the rule: The previous + # non-empty line has the paramters of a function header that are indented + # 4 spaces (because they did not fit in a 80 column line when placed on + # the same line as the function name). We also check for the case where + # the previous line is indented 6 spaces, which may happen when the + # initializers of a constructor do not fit into a 80 column line. + exception = False + if Match(r' {6}\w', prev_line): # Initializer list? + # We are looking for the opening column of initializer list, which + # should be indented 4 spaces to cause 6 space indentation afterwards. + search_position = linenum-2 + while (search_position >= 0 + and Match(r' {6}\w', elided[search_position])): + search_position -= 1 + exception = (search_position >= 0 + and elided[search_position][:5] == ' :') + else: + # Search for the function arguments or an initializer list. We use a + # simple heuristic here: If the line is indented 4 spaces; and we have a + # closing paren, without the opening paren, followed by an opening brace + # or colon (for initializer lists) we assume that it is the last line of + # a function header. If we have a colon indented 4 spaces, it is an + # initializer list. + exception = (Match(r' {4}\w[^\(]*\)\s*(const\s*)?(\{\s*$|:)', + prev_line) + or Match(r' {4}:', prev_line)) + + if not exception: + error(filename, linenum, 'whitespace/blank_line', 2, + 'Blank line at the start of a code block. Is this needed?') + # This doesn't ignore whitespace at the end of a namespace block + # because that is too hard without pairing open/close braces; + # however, a special exception is made for namespace closing + # brackets which have a comment containing "namespace". + # + # Also, ignore blank lines at the end of a block in a long if-else + # chain, like this: + # if (condition1) { + # // Something followed by a blank line + # + # } else if (condition2) { + # // Something else + # } + if linenum + 1 < clean_lines.NumLines(): + next_line = raw[linenum + 1] + if (next_line + and Match(r'\s*}', next_line) + and next_line.find('namespace') == -1 + and next_line.find('} else ') == -1): + error(filename, linenum, 'whitespace/blank_line', 3, + 'Blank line at the end of a code block. Is this needed?') + + # Next, we complain if there's a comment too near the text + commentpos = line.find('//') + if commentpos != -1: + # Check if the // may be in quotes. If so, ignore it + # Comparisons made explicit for clarity -- pylint: disable-msg=C6403 + if (line.count('"', 0, commentpos) - + line.count('\\"', 0, commentpos)) % 2 == 0: # not in quotes + # Allow one space for new scopes, two spaces otherwise: + if (not Match(r'^\s*{ //', line) and + ((commentpos >= 1 and + line[commentpos-1] not in string.whitespace) or + (commentpos >= 2 and + line[commentpos-2] not in string.whitespace))): + error(filename, linenum, 'whitespace/comments', 2, + 'At least two spaces is best between code and comments') + # There should always be a space between the // and the comment + commentend = commentpos + 2 + if commentend < len(line) and not line[commentend] == ' ': + # but some lines are exceptions -- e.g. if they're big + # comment delimiters like: + # //---------------------------------------------------------- + # or they begin with multiple slashes followed by a space: + # //////// Header comment + match = (Search(r'[=/-]{4,}\s*$', line[commentend:]) or + Search(r'^/+ ', line[commentend:])) + if not match: + error(filename, linenum, 'whitespace/comments', 4, + 'Should have a space between // and comment') + CheckComment(line[commentpos:], filename, linenum, error) + + line = clean_lines.elided[linenum] # get rid of comments and strings + + # Don't try to do spacing checks for operator methods + line = re.sub(r'operator(==|!=|<|<<|<=|>=|>>|>)\(', 'operator\(', line) + + # We allow no-spaces around = within an if: "if ( (a=Foo()) == 0 )". + # Otherwise not. Note we only check for non-spaces on *both* sides; + # sometimes people put non-spaces on one side when aligning ='s among + # many lines (not that this is behavior that I approve of...) + if Search(r'[\w.]=[\w.]', line) and not Search(r'\b(if|while) ', line): + error(filename, linenum, 'whitespace/operators', 4, + 'Missing spaces around =') + + # It's ok not to have spaces around binary operators like + - * /, but if + # there's too little whitespace, we get concerned. It's hard to tell, + # though, so we punt on this one for now. TODO. + + # You should always have whitespace around binary operators. + # Alas, we can't test < or > because they're legitimately used sans spaces + # (a->b, vector a). The only time we can tell is a < with no >, and + # only if it's not template params list spilling into the next line. + match = Search(r'[^<>=!\s](==|!=|<=|>=)[^<>=!\s]', line) + if not match: + # Note that while it seems that the '<[^<]*' term in the following + # regexp could be simplified to '<.*', which would indeed match + # the same class of strings, the [^<] means that searching for the + # regexp takes linear rather than quadratic time. + if not Search(r'<[^<]*,\s*$', line): # template params spill + match = Search(r'[^<>=!\s](<)[^<>=!\s]([^>]|->)*$', line) + if match: + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around %s' % match.group(1)) + # We allow no-spaces around << and >> when used like this: 10<<20, but + # not otherwise (particularly, not when used as streams) + match = Search(r'[^0-9\s](<<|>>)[^0-9\s]', line) + if match: + error(filename, linenum, 'whitespace/operators', 3, + 'Missing spaces around %s' % match.group(1)) + + # There shouldn't be space around unary operators + match = Search(r'(!\s|~\s|[\s]--[\s;]|[\s]\+\+[\s;])', line) + if match: + error(filename, linenum, 'whitespace/operators', 4, + 'Extra space for operator %s' % match.group(1)) + + # A pet peeve of mine: no spaces after an if, while, switch, or for + match = Search(r' (if\(|for\(|while\(|switch\()', line) + if match: + error(filename, linenum, 'whitespace/parens', 5, + 'Missing space before ( in %s' % match.group(1)) + + # For if/for/while/switch, the left and right parens should be + # consistent about how many spaces are inside the parens, and + # there should either be zero or one spaces inside the parens. + # We don't want: "if ( foo)" or "if ( foo )". + # Exception: "for ( ; foo; bar)" and "for (foo; bar; )" are allowed. + match = Search(r'\b(if|for|while|switch)\s*' + r'\(([ ]*)(.).*[^ ]+([ ]*)\)\s*{\s*$', + line) + if match: + if len(match.group(2)) != len(match.group(4)): + if not (match.group(3) == ';' and + len(match.group(2)) == 1 + len(match.group(4)) or + not match.group(2) and Search(r'\bfor\s*\(.*; \)', line)): + error(filename, linenum, 'whitespace/parens', 5, + 'Mismatching spaces inside () in %s' % match.group(1)) + if not len(match.group(2)) in [0, 1]: + error(filename, linenum, 'whitespace/parens', 5, + 'Should have zero or one spaces inside ( and ) in %s' % + match.group(1)) + + # You should always have a space after a comma (either as fn arg or operator) + if Search(r',[^\s]', line): + error(filename, linenum, 'whitespace/comma', 3, + 'Missing space after ,') + + # Next we will look for issues with function calls. + CheckSpacingForFunctionCall(filename, line, linenum, error) + + # Except after an opening paren, you should have spaces before your braces. + # And since you should never have braces at the beginning of a line, this is + # an easy test. + if Search(r'[^ (]{', line): + error(filename, linenum, 'whitespace/braces', 5, + 'Missing space before {') + + # Make sure '} else {' has spaces. + if Search(r'}else', line): + error(filename, linenum, 'whitespace/braces', 5, + 'Missing space before else') + + # You shouldn't have spaces before your brackets, except maybe after + # 'delete []' or 'new char * []'. + if Search(r'\w\s+\[', line) and not Search(r'delete\s+\[', line): + error(filename, linenum, 'whitespace/braces', 5, + 'Extra space before [') + + # You shouldn't have a space before a semicolon at the end of the line. + # There's a special case for "for" since the style guide allows space before + # the semicolon there. + if Search(r':\s*;\s*$', line): + error(filename, linenum, 'whitespace/semicolon', 5, + 'Semicolon defining empty statement. Use { } instead.') + elif Search(r'^\s*;\s*$', line): + error(filename, linenum, 'whitespace/semicolon', 5, + 'Line contains only semicolon. If this should be an empty statement, ' + 'use { } instead.') + elif (Search(r'\s+;\s*$', line) and + not Search(r'\bfor\b', line)): + error(filename, linenum, 'whitespace/semicolon', 5, + 'Extra space before last semicolon. If this should be an empty ' + 'statement, use { } instead.') + + +def GetPreviousNonBlankLine(clean_lines, linenum): + """Return the most recent non-blank line and its line number. + + Args: + clean_lines: A CleansedLines instance containing the file contents. + linenum: The number of the line to check. + + Returns: + A tuple with two elements. The first element is the contents of the last + non-blank line before the current line, or the empty string if this is the + first non-blank line. The second is the line number of that line, or -1 + if this is the first non-blank line. + """ + + prevlinenum = linenum - 1 + while prevlinenum >= 0: + prevline = clean_lines.elided[prevlinenum] + if not IsBlankLine(prevline): # if not a blank line... + return (prevline, prevlinenum) + prevlinenum -= 1 + return ('', -1) + + +def CheckBraces(filename, clean_lines, linenum, error): + """Looks for misplaced braces (e.g. at the end of line). + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + + line = clean_lines.elided[linenum] # get rid of comments and strings + + if Match(r'\s*{\s*$', line): + # We allow an open brace to start a line in the case where someone + # is using braces in a block to explicitly create a new scope, + # which is commonly used to control the lifetime of + # stack-allocated variables. We don't detect this perfectly: we + # just don't complain if the last non-whitespace character on the + # previous non-blank line is ';', ':', '{', or '}'. + prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] + if not Search(r'[;:}{]\s*$', prevline): + error(filename, linenum, 'whitespace/braces', 4, + '{ should almost always be at the end of the previous line') + + # An else clause should be on the same line as the preceding closing brace. + if Match(r'\s*else\s*', line): + prevline = GetPreviousNonBlankLine(clean_lines, linenum)[0] + if Match(r'\s*}\s*$', prevline): + error(filename, linenum, 'whitespace/newline', 4, + 'An else should appear on the same line as the preceding }') + + # If braces come on one side of an else, they should be on both. + # However, we have to worry about "else if" that spans multiple lines! + if Search(r'}\s*else[^{]*$', line) or Match(r'[^}]*else\s*{', line): + if Search(r'}\s*else if([^{]*)$', line): # could be multi-line if + # find the ( after the if + pos = line.find('else if') + pos = line.find('(', pos) + if pos > 0: + (endline, _, endpos) = CloseExpression(clean_lines, linenum, pos) + if endline[endpos:].find('{') == -1: # must be brace after if + error(filename, linenum, 'readability/braces', 5, + 'If an else has a brace on one side, it should have it on both') + else: # common case: else not followed by a multi-line if + error(filename, linenum, 'readability/braces', 5, + 'If an else has a brace on one side, it should have it on both') + + # Likewise, an else should never have the else clause on the same line + if Search(r'\belse [^\s{]', line) and not Search(r'\belse if\b', line): + error(filename, linenum, 'whitespace/newline', 4, + 'Else clause should never be on same line as else (use 2 lines)') + + # In the same way, a do/while should never be on one line + if Match(r'\s*do [^\s{]', line): + error(filename, linenum, 'whitespace/newline', 4, + 'do/while clauses should not be on a single line') + + # Braces shouldn't be followed by a ; unless they're defining a struct + # or initializing an array. + # We can't tell in general, but we can for some common cases. + prevlinenum = linenum + while True: + (prevline, prevlinenum) = GetPreviousNonBlankLine(clean_lines, prevlinenum) + if Match(r'\s+{.*}\s*;', line) and not prevline.count(';'): + line = prevline + line + else: + break + if (Search(r'{.*}\s*;', line) and + line.count('{') == line.count('}') and + not Search(r'struct|class|enum|\s*=\s*{', line)): + error(filename, linenum, 'readability/braces', 4, + "You don't need a ; after a }") + + +def ReplaceableCheck(operator, macro, line): + """Determine whether a basic CHECK can be replaced with a more specific one. + + For example suggest using CHECK_EQ instead of CHECK(a == b) and + similarly for CHECK_GE, CHECK_GT, CHECK_LE, CHECK_LT, CHECK_NE. + + Args: + operator: The C++ operator used in the CHECK. + macro: The CHECK or EXPECT macro being called. + line: The current source line. + + Returns: + True if the CHECK can be replaced with a more specific one. + """ + + # This matches decimal and hex integers, strings, and chars (in that order). + match_constant = r'([-+]?(\d+|0[xX][0-9a-fA-F]+)[lLuU]{0,3}|".*"|\'.*\')' + + # Expression to match two sides of the operator with something that + # looks like a literal, since CHECK(x == iterator) won't compile. + # This means we can't catch all the cases where a more specific + # CHECK is possible, but it's less annoying than dealing with + # extraneous warnings. + match_this = (r'\s*' + macro + r'\((\s*' + + match_constant + r'\s*' + operator + r'[^<>].*|' + r'.*[^<>]' + operator + r'\s*' + match_constant + + r'\s*\))') + + # Don't complain about CHECK(x == NULL) or similar because + # CHECK_EQ(x, NULL) won't compile (requires a cast). + # Also, don't complain about more complex boolean expressions + # involving && or || such as CHECK(a == b || c == d). + return Match(match_this, line) and not Search(r'NULL|&&|\|\|', line) + + +def CheckCheck(filename, clean_lines, linenum, error): + """Checks the use of CHECK and EXPECT macros. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + error: The function to call with any errors found. + """ + + # Decide the set of replacement macros that should be suggested + raw_lines = clean_lines.raw_lines + current_macro = '' + for macro in _CHECK_MACROS: + if raw_lines[linenum].find(macro) >= 0: + current_macro = macro + break + if not current_macro: + # Don't waste time here if line doesn't contain 'CHECK' or 'EXPECT' + return + + line = clean_lines.elided[linenum] # get rid of comments and strings + + # Encourage replacing plain CHECKs with CHECK_EQ/CHECK_NE/etc. + for operator in ['==', '!=', '>=', '>', '<=', '<']: + if ReplaceableCheck(operator, current_macro, line): + error(filename, linenum, 'readability/check', 2, + 'Consider using %s instead of %s(a %s b)' % ( + _CHECK_REPLACEMENT[current_macro][operator], + current_macro, operator)) + break + + +def GetLineWidth(line): + """Determines the width of the line in column positions. + + Args: + line: A string, which may be a Unicode string. + + Returns: + The width of the line in column positions, accounting for Unicode + combining characters and wide characters. + """ + if isinstance(line, unicode): + width = 0 + for c in unicodedata.normalize('NFC', line): + if unicodedata.east_asian_width(c) in ('W', 'F'): + width += 2 + elif not unicodedata.combining(c): + width += 1 + return width + else: + return len(line) + + +def CheckStyle(filename, clean_lines, linenum, file_extension, error): + """Checks rules from the 'C++ style rules' section of cppguide.html. + + Most of these rules are hard to test (naming, comment style), but we + do what we can. In particular we check for 2-space indents, line lengths, + tab usage, spaces inside code, etc. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + file_extension: The extension (without the dot) of the filename. + error: The function to call with any errors found. + """ + + raw_lines = clean_lines.raw_lines + line = raw_lines[linenum] + + if line.find('\t') != -1: + error(filename, linenum, 'whitespace/tab', 1, + 'Tab found; better to use spaces') + + # One or three blank spaces at the beginning of the line is weird; it's + # hard to reconcile that with 2-space indents. + # NOTE: here are the conditions rob pike used for his tests. Mine aren't + # as sophisticated, but it may be worth becoming so: RLENGTH==initial_spaces + # if(RLENGTH > 20) complain = 0; + # if(match($0, " +(error|private|public|protected):")) complain = 0; + # if(match(prev, "&& *$")) complain = 0; + # if(match(prev, "\\|\\| *$")) complain = 0; + # if(match(prev, "[\",=><] *$")) complain = 0; + # if(match($0, " <<")) complain = 0; + # if(match(prev, " +for \\(")) complain = 0; + # if(prevodd && match(prevprev, " +for \\(")) complain = 0; + initial_spaces = 0 + cleansed_line = clean_lines.elided[linenum] + while initial_spaces < len(line) and line[initial_spaces] == ' ': + initial_spaces += 1 + if line and line[-1].isspace(): + error(filename, linenum, 'whitespace/end_of_line', 4, + 'Line ends in whitespace. Consider deleting these extra spaces.') + # There are certain situations we allow one space, notably for labels + elif ((initial_spaces == 1 or initial_spaces == 3) and + not Match(r'\s*\w+\s*:\s*$', cleansed_line)): + error(filename, linenum, 'whitespace/indent', 3, + 'Weird number of spaces at line-start. ' + 'Are you using a 2-space indent?') + # Labels should always be indented at least one space. + elif not initial_spaces and line[:2] != '//' and Search(r'[^:]:\s*$', + line): + error(filename, linenum, 'whitespace/labels', 4, + 'Labels should always be indented at least one space. ' + 'If this is a member-initializer list in a constructor, ' + 'the colon should be on the line after the definition header.') + + # Check if the line is a header guard. + is_header_guard = False + if file_extension == 'h': + cppvar = GetHeaderGuardCPPVariable(filename) + if (line.startswith('#ifndef %s' % cppvar) or + line.startswith('#define %s' % cppvar) or + line.startswith('#endif // %s' % cppvar)): + is_header_guard = True + # #include lines and header guards can be long, since there's no clean way to + # split them. + # + # URLs can be long too. It's possible to split these, but it makes them + # harder to cut&paste. + if (not line.startswith('#include') and not is_header_guard and + not Match(r'^\s*//.*http(s?)://\S*$', line)): + line_width = GetLineWidth(line) + if line_width > 100: + error(filename, linenum, 'whitespace/line_length', 4, + 'Lines should very rarely be longer than 100 characters') + elif line_width > 80: + error(filename, linenum, 'whitespace/line_length', 2, + 'Lines should be <= 80 characters long') + + if (cleansed_line.count(';') > 1 and + # for loops are allowed two ;'s (and may run over two lines). + cleansed_line.find('for') == -1 and + (GetPreviousNonBlankLine(clean_lines, linenum)[0].find('for') == -1 or + GetPreviousNonBlankLine(clean_lines, linenum)[0].find(';') != -1) and + # It's ok to have many commands in a switch case that fits in 1 line + not ((cleansed_line.find('case ') != -1 or + cleansed_line.find('default:') != -1) and + cleansed_line.find('break;') != -1)): + error(filename, linenum, 'whitespace/newline', 4, + 'More than one command on the same line') + + # Some more style checks + CheckBraces(filename, clean_lines, linenum, error) + CheckSpacing(filename, clean_lines, linenum, error) + CheckCheck(filename, clean_lines, linenum, error) + + +_RE_PATTERN_INCLUDE_NEW_STYLE = re.compile(r'#include +"[^/]+\.h"') +_RE_PATTERN_INCLUDE = re.compile(r'^\s*#\s*include\s*([<"])([^>"]*)[>"].*$') +# Matches the first component of a filename delimited by -s and _s. That is: +# _RE_FIRST_COMPONENT.match('foo').group(0) == 'foo' +# _RE_FIRST_COMPONENT.match('foo.cc').group(0) == 'foo' +# _RE_FIRST_COMPONENT.match('foo-bar_baz.cc').group(0) == 'foo' +# _RE_FIRST_COMPONENT.match('foo_bar-baz.cc').group(0) == 'foo' +_RE_FIRST_COMPONENT = re.compile(r'^[^-_.]+') + + +def _DropCommonSuffixes(filename): + """Drops common suffixes like _test.cc or -inl.h from filename. + + For example: + >>> _DropCommonSuffixes('foo/foo-inl.h') + 'foo/foo' + >>> _DropCommonSuffixes('foo/bar/foo.cc') + 'foo/bar/foo' + >>> _DropCommonSuffixes('foo/foo_internal.h') + 'foo/foo' + >>> _DropCommonSuffixes('foo/foo_unusualinternal.h') + 'foo/foo_unusualinternal' + + Args: + filename: The input filename. + + Returns: + The filename with the common suffix removed. + """ + for suffix in ('test.cc', 'regtest.cc', 'unittest.cc', + 'inl.h', 'impl.h', 'internal.h'): + if (filename.endswith(suffix) and len(filename) > len(suffix) and + filename[-len(suffix) - 1] in ('-', '_')): + return filename[:-len(suffix) - 1] + return os.path.splitext(filename)[0] + + +def _IsTestFilename(filename): + """Determines if the given filename has a suffix that identifies it as a test. + + Args: + filename: The input filename. + + Returns: + True if 'filename' looks like a test, False otherwise. + """ + if (filename.endswith('_test.cc') or + filename.endswith('_unittest.cc') or + filename.endswith('_regtest.cc')): + return True + else: + return False + + +def _ClassifyInclude(fileinfo, include, is_system): + """Figures out what kind of header 'include' is. + + Args: + fileinfo: The current file cpplint is running over. A FileInfo instance. + include: The path to a #included file. + is_system: True if the #include used <> rather than "". + + Returns: + One of the _XXX_HEADER constants. + + For example: + >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'stdio.h', True) + _C_SYS_HEADER + >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'string', True) + _CPP_SYS_HEADER + >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'foo/foo.h', False) + _LIKELY_MY_HEADER + >>> _ClassifyInclude(FileInfo('foo/foo_unknown_extension.cc'), + ... 'bar/foo_other_ext.h', False) + _POSSIBLE_MY_HEADER + >>> _ClassifyInclude(FileInfo('foo/foo.cc'), 'foo/bar.h', False) + _OTHER_HEADER + """ + # This is a list of all standard c++ header files, except + # those already checked for above. + is_stl_h = include in _STL_HEADERS + is_cpp_h = is_stl_h or include in _CPP_HEADERS + + if is_system: + if is_cpp_h: + return _CPP_SYS_HEADER + else: + return _C_SYS_HEADER + + # If the target file and the include we're checking share a + # basename when we drop common extensions, and the include + # lives in . , then it's likely to be owned by the target file. + target_dir, target_base = ( + os.path.split(_DropCommonSuffixes(fileinfo.RepositoryName()))) + include_dir, include_base = os.path.split(_DropCommonSuffixes(include)) + if target_base == include_base and ( + include_dir == target_dir or + include_dir == os.path.normpath(target_dir + '/../public')): + return _LIKELY_MY_HEADER + + # If the target and include share some initial basename + # component, it's possible the target is implementing the + # include, so it's allowed to be first, but we'll never + # complain if it's not there. + target_first_component = _RE_FIRST_COMPONENT.match(target_base) + include_first_component = _RE_FIRST_COMPONENT.match(include_base) + if (target_first_component and include_first_component and + target_first_component.group(0) == + include_first_component.group(0)): + return _POSSIBLE_MY_HEADER + + return _OTHER_HEADER + + + +def CheckIncludeLine(filename, clean_lines, linenum, include_state, error): + """Check rules that are applicable to #include lines. + + Strings on #include lines are NOT removed from elided line, to make + certain tasks easier. However, to prevent false positives, checks + applicable to #include lines in CheckLanguage must be put here. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + include_state: An _IncludeState instance in which the headers are inserted. + error: The function to call with any errors found. + """ + fileinfo = FileInfo(filename) + + line = clean_lines.lines[linenum] + + # "include" should use the new style "foo/bar.h" instead of just "bar.h" + if _RE_PATTERN_INCLUDE_NEW_STYLE.search(line): + error(filename, linenum, 'build/include', 4, + 'Include the directory when naming .h files') + + # we shouldn't include a file more than once. actually, there are a + # handful of instances where doing so is okay, but in general it's + # not. + match = _RE_PATTERN_INCLUDE.search(line) + if match: + include = match.group(2) + is_system = (match.group(1) == '<') + if include in include_state: + error(filename, linenum, 'build/include', 4, + '"%s" already included at %s:%s' % + (include, filename, include_state[include])) + else: + include_state[include] = linenum + + # We want to ensure that headers appear in the right order: + # 1) for foo.cc, foo.h (preferred location) + # 2) c system files + # 3) cpp system files + # 4) for foo.cc, foo.h (deprecated location) + # 5) other google headers + # + # We classify each include statement as one of those 5 types + # using a number of techniques. The include_state object keeps + # track of the highest type seen, and complains if we see a + # lower type after that. + error_message = include_state.CheckNextIncludeOrder( + _ClassifyInclude(fileinfo, include, is_system)) + if error_message: + error(filename, linenum, 'build/include_order', 4, + '%s. Should be: %s.h, c system, c++ system, other.' % + (error_message, fileinfo.BaseName())) + if not include_state.IsInAlphabeticalOrder(include): + error(filename, linenum, 'build/include_alpha', 4, + 'Include "%s" not in alphabetical order' % include) + + # Look for any of the stream classes that are part of standard C++. + match = _RE_PATTERN_INCLUDE.match(line) + if match: + include = match.group(2) + if Match(r'(f|ind|io|i|o|parse|pf|stdio|str|)?stream$', include): + # Many unit tests use cout, so we exempt them. + if not _IsTestFilename(filename): + error(filename, linenum, 'readability/streams', 3, + 'Streams are highly discouraged.') + +def CheckLanguage(filename, clean_lines, linenum, file_extension, include_state, + error): + """Checks rules from the 'C++ language rules' section of cppguide.html. + + Some of these rules are hard to test (function overloading, using + uint32 inappropriately), but we do the best we can. + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + linenum: The number of the line to check. + file_extension: The extension (without the dot) of the filename. + include_state: An _IncludeState instance in which the headers are inserted. + error: The function to call with any errors found. + """ + # If the line is empty or consists of entirely a comment, no need to + # check it. + line = clean_lines.elided[linenum] + if not line: + return + + match = _RE_PATTERN_INCLUDE.search(line) + if match: + CheckIncludeLine(filename, clean_lines, linenum, include_state, error) + return + + # Create an extended_line, which is the concatenation of the current and + # next lines, for more effective checking of code that may span more than one + # line. + if linenum + 1 < clean_lines.NumLines(): + extended_line = line + clean_lines.elided[linenum + 1] + else: + extended_line = line + + # Make Windows paths like Unix. + fullname = os.path.abspath(filename).replace('\\', '/') + + # TODO(unknown): figure out if they're using default arguments in fn proto. + + # Check for non-const references in functions. This is tricky because & + # is also used to take the address of something. We allow <> for templates, + # (ignoring whatever is between the braces) and : for classes. + # These are complicated re's. They try to capture the following: + # paren (for fn-prototype start), typename, &, varname. For the const + # version, we're willing for const to be before typename or after + # Don't check the implemention on same line. + fnline = line.split('{', 1)[0] + if (len(re.findall(r'\([^()]*\b(?:[\w:]|<[^()]*>)+(\s?&|&\s?)\w+', fnline)) > + len(re.findall(r'\([^()]*\bconst\s+(?:typename\s+)?(?:struct\s+)?' + r'(?:[\w:]|<[^()]*>)+(\s?&|&\s?)\w+', fnline)) + + len(re.findall(r'\([^()]*\b(?:[\w:]|<[^()]*>)+\s+const(\s?&|&\s?)[\w]+', + fnline))): + + # We allow non-const references in a few standard places, like functions + # called "swap()" or iostream operators like "<<" or ">>". + if not Search( + r'(swap|Swap|operator[<>][<>])\s*\(\s*(?:[\w:]|<.*>)+\s*&', + fnline): + error(filename, linenum, 'runtime/references', 2, + 'Is this a non-const reference? ' + 'If so, make const or use a pointer.') + + # Check to see if they're using an conversion function cast. + # I just try to capture the most common basic types, though there are more. + # Parameterless conversion functions, such as bool(), are allowed as they are + # probably a member operator declaration or default constructor. + match = Search( + r'(\bnew\s+)?\b' # Grab 'new' operator, if it's there + r'(int|float|double|bool|char|int32|uint32|int64|uint64)\([^)]', line) + if match: + # gMock methods are defined using some variant of MOCK_METHODx(name, type) + # where type may be float(), int(string), etc. Without context they are + # virtually indistinguishable from int(x) casts. + if (match.group(1) is None and # If new operator, then this isn't a cast + not Match(r'^\s*MOCK_(CONST_)?METHOD\d+(_T)?\(', line)): + error(filename, linenum, 'readability/casting', 4, + 'Using deprecated casting style. ' + 'Use static_cast<%s>(...) instead' % + match.group(2)) + + CheckCStyleCast(filename, linenum, line, clean_lines.raw_lines[linenum], + 'static_cast', + r'\((int|float|double|bool|char|u?int(16|32|64))\)', + error) + # This doesn't catch all cases. Consider (const char * const)"hello". + CheckCStyleCast(filename, linenum, line, clean_lines.raw_lines[linenum], + 'reinterpret_cast', r'\((\w+\s?\*+\s?)\)', error) + + # In addition, we look for people taking the address of a cast. This + # is dangerous -- casts can assign to temporaries, so the pointer doesn't + # point where you think. + if Search( + r'(&\([^)]+\)[\w(])|(&(static|dynamic|reinterpret)_cast\b)', line): + error(filename, linenum, 'runtime/casting', 4, + ('Are you taking an address of a cast? ' + 'This is dangerous: could be a temp var. ' + 'Take the address before doing the cast, rather than after')) + + # Check for people declaring static/global STL strings at the top level. + # This is dangerous because the C++ language does not guarantee that + # globals with constructors are initialized before the first access. + match = Match( + r'((?:|static +)(?:|const +))string +([a-zA-Z0-9_:]+)\b(.*)', + line) + # Make sure it's not a function. + # Function template specialization looks like: "string foo(...". + # Class template definitions look like: "string Foo::Method(...". + if match and not Match(r'\s*(<.*>)?(::[a-zA-Z0-9_]+)?\s*\(([^"]|$)', + match.group(3)): + error(filename, linenum, 'runtime/string', 4, + 'For a static/global string constant, use a C style string instead: ' + '"%schar %s[]".' % + (match.group(1), match.group(2))) + + # Check that we're not using RTTI outside of testing code. + if Search(r'\bdynamic_cast<', line) and not _IsTestFilename(filename): + error(filename, linenum, 'runtime/rtti', 5, + 'Do not use dynamic_cast<>. If you need to cast within a class ' + "hierarchy, use static_cast<> to upcast. Google doesn't support " + 'RTTI.') + + if Search(r'\b([A-Za-z0-9_]*_)\(\1\)', line): + error(filename, linenum, 'runtime/init', 4, + 'You seem to be initializing a member variable with itself.') + + if file_extension == 'h': + # TODO(unknown): check that 1-arg constructors are explicit. + # How to tell it's a constructor? + # (handled in CheckForNonStandardConstructs for now) + # TODO(unknown): check that classes have DISALLOW_EVIL_CONSTRUCTORS + # (level 1 error) + pass + + # Check if people are using the verboten C basic types. The only exception + # we regularly allow is "unsigned short port" for port. + if Search(r'\bshort port\b', line): + if not Search(r'\bunsigned short port\b', line): + error(filename, linenum, 'runtime/int', 4, + 'Use "unsigned short" for ports, not "short"') + else: + match = Search(r'\b(short|long(?! +double)|long long)\b', line) + if match: + error(filename, linenum, 'runtime/int', 4, + 'Use int16/int64/etc, rather than the C type %s' % match.group(1)) + + # When snprintf is used, the second argument shouldn't be a literal. + match = Search(r'snprintf\s*\(([^,]*),\s*([0-9]*)\s*,', line) + if match: + error(filename, linenum, 'runtime/printf', 3, + 'If you can, use sizeof(%s) instead of %s as the 2nd arg ' + 'to snprintf.' % (match.group(1), match.group(2))) + + # Check if some verboten C functions are being used. + if Search(r'\bsprintf\b', line): + error(filename, linenum, 'runtime/printf', 5, + 'Never use sprintf. Use snprintf instead.') + match = Search(r'\b(strcpy|strcat)\b', line) + if match: + error(filename, linenum, 'runtime/printf', 4, + 'Almost always, snprintf is better than %s' % match.group(1)) + + if Search(r'\bsscanf\b', line): + error(filename, linenum, 'runtime/printf', 1, + 'sscanf can be ok, but is slow and can overflow buffers.') + + # Check if some verboten operator overloading is going on + # TODO(unknown): catch out-of-line unary operator&: + # class X {}; + # int operator&(const X& x) { return 42; } // unary operator& + # The trick is it's hard to tell apart from binary operator&: + # class Y { int operator&(const Y& x) { return 23; } }; // binary operator& + if Search(r'\boperator\s*&\s*\(\s*\)', line): + error(filename, linenum, 'runtime/operator', 4, + 'Unary operator& is dangerous. Do not use it.') + + # Check for suspicious usage of "if" like + # } if (a == b) { + if Search(r'\}\s*if\s*\(', line): + error(filename, linenum, 'readability/braces', 4, + 'Did you mean "else if"? If not, start a new line for "if".') + + # Check for potential format string bugs like printf(foo). + # We constrain the pattern not to pick things like DocidForPrintf(foo). + # Not perfect but it can catch printf(foo.c_str()) and printf(foo->c_str()) + match = re.search(r'\b((?:string)?printf)\s*\(([\w.\->()]+)\)', line, re.I) + if match: + error(filename, linenum, 'runtime/printf', 4, + 'Potential format string bug. Do %s("%%s", %s) instead.' + % (match.group(1), match.group(2))) + + # Check for potential memset bugs like memset(buf, sizeof(buf), 0). + match = Search(r'memset\s*\(([^,]*),\s*([^,]*),\s*0\s*\)', line) + if match and not Match(r"^''|-?[0-9]+|0x[0-9A-Fa-f]$", match.group(2)): + error(filename, linenum, 'runtime/memset', 4, + 'Did you mean "memset(%s, 0, %s)"?' + % (match.group(1), match.group(2))) + + if Search(r'\busing namespace\b', line): + error(filename, linenum, 'build/namespaces', 5, + 'Do not use namespace using-directives. ' + 'Use using-declarations instead.') + + # Detect variable-length arrays. + match = Match(r'\s*(.+::)?(\w+) [a-z]\w*\[(.+)];', line) + if (match and match.group(2) != 'return' and match.group(2) != 'delete' and + match.group(3).find(']') == -1): + # Split the size using space and arithmetic operators as delimiters. + # If any of the resulting tokens are not compile time constants then + # report the error. + tokens = re.split(r'\s|\+|\-|\*|\/|<<|>>]', match.group(3)) + is_const = True + skip_next = False + for tok in tokens: + if skip_next: + skip_next = False + continue + + if Search(r'sizeof\(.+\)', tok): continue + if Search(r'arraysize\(\w+\)', tok): continue + + tok = tok.lstrip('(') + tok = tok.rstrip(')') + if not tok: continue + if Match(r'\d+', tok): continue + if Match(r'0[xX][0-9a-fA-F]+', tok): continue + if Match(r'k[A-Z0-9]\w*', tok): continue + if Match(r'(.+::)?k[A-Z0-9]\w*', tok): continue + if Match(r'(.+::)?[A-Z][A-Z0-9_]*', tok): continue + # A catch all for tricky sizeof cases, including 'sizeof expression', + # 'sizeof(*type)', 'sizeof(const type)', 'sizeof(struct StructName)' + # requires skipping the next token becasue we split on ' ' and '*'. + if tok.startswith('sizeof'): + skip_next = True + continue + is_const = False + break + if not is_const: + error(filename, linenum, 'runtime/arrays', 1, + 'Do not use variable-length arrays. Use an appropriately named ' + "('k' followed by CamelCase) compile-time constant for the size.") + + # If DISALLOW_EVIL_CONSTRUCTORS, DISALLOW_COPY_AND_ASSIGN, or + # DISALLOW_IMPLICIT_CONSTRUCTORS is present, then it should be the last thing + # in the class declaration. + match = Match( + (r'\s*' + r'(DISALLOW_(EVIL_CONSTRUCTORS|COPY_AND_ASSIGN|IMPLICIT_CONSTRUCTORS))' + r'\(.*\);$'), + line) + if match and linenum + 1 < clean_lines.NumLines(): + next_line = clean_lines.elided[linenum + 1] + if not Search(r'^\s*};', next_line): + error(filename, linenum, 'readability/constructors', 3, + match.group(1) + ' should be the last thing in the class') + + # Check for use of unnamed namespaces in header files. Registration + # macros are typically OK, so we allow use of "namespace {" on lines + # that end with backslashes. + if (file_extension == 'h' + and Search(r'\bnamespace\s*{', line) + and line[-1] != '\\'): + error(filename, linenum, 'build/namespaces', 4, + 'Do not use unnamed namespaces in header files. See ' + 'http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml#Namespaces' + ' for more information.') + + +def CheckCStyleCast(filename, linenum, line, raw_line, cast_type, pattern, + error): + """Checks for a C-style cast by looking for the pattern. + + This also handles sizeof(type) warnings, due to similarity of content. + + Args: + filename: The name of the current file. + linenum: The number of the line to check. + line: The line of code to check. + raw_line: The raw line of code to check, with comments. + cast_type: The string for the C++ cast to recommend. This is either + reinterpret_cast or static_cast, depending. + pattern: The regular expression used to find C-style casts. + error: The function to call with any errors found. + """ + match = Search(pattern, line) + if not match: + return + + # e.g., sizeof(int) + sizeof_match = Match(r'.*sizeof\s*$', line[0:match.start(1) - 1]) + if sizeof_match: + error(filename, linenum, 'runtime/sizeof', 1, + 'Using sizeof(type). Use sizeof(varname) instead if possible') + return + + remainder = line[match.end(0):] + + # The close paren is for function pointers as arguments to a function. + # eg, void foo(void (*bar)(int)); + # The semicolon check is a more basic function check; also possibly a + # function pointer typedef. + # eg, void foo(int); or void foo(int) const; + # The equals check is for function pointer assignment. + # eg, void *(*foo)(int) = ... + # + # Right now, this will only catch cases where there's a single argument, and + # it's unnamed. It should probably be expanded to check for multiple + # arguments with some unnamed. + function_match = Match(r'\s*(\)|=|(const)?\s*(;|\{|throw\(\)))', remainder) + if function_match: + if (not function_match.group(3) or + function_match.group(3) == ';' or + raw_line.find('/*') < 0): + error(filename, linenum, 'readability/function', 3, + 'All parameters should be named in a function') + return + + # At this point, all that should be left is actual casts. + error(filename, linenum, 'readability/casting', 4, + 'Using C-style cast. Use %s<%s>(...) instead' % + (cast_type, match.group(1))) + + +_HEADERS_CONTAINING_TEMPLATES = ( + ('', ('deque',)), + ('', ('unary_function', 'binary_function', + 'plus', 'minus', 'multiplies', 'divides', 'modulus', + 'negate', + 'equal_to', 'not_equal_to', 'greater', 'less', + 'greater_equal', 'less_equal', + 'logical_and', 'logical_or', 'logical_not', + 'unary_negate', 'not1', 'binary_negate', 'not2', + 'bind1st', 'bind2nd', + 'pointer_to_unary_function', + 'pointer_to_binary_function', + 'ptr_fun', + 'mem_fun_t', 'mem_fun', 'mem_fun1_t', 'mem_fun1_ref_t', + 'mem_fun_ref_t', + 'const_mem_fun_t', 'const_mem_fun1_t', + 'const_mem_fun_ref_t', 'const_mem_fun1_ref_t', + 'mem_fun_ref', + )), + ('', ('numeric_limits',)), + ('', ('list',)), + ('', ('map', 'multimap',)), + ('', ('allocator',)), + ('', ('queue', 'priority_queue',)), + ('', ('set', 'multiset',)), + ('', ('stack',)), + ('', ('char_traits', 'basic_string',)), + ('', ('pair',)), + ('', ('vector',)), + + # gcc extensions. + # Note: std::hash is their hash, ::hash is our hash + ('', ('hash_map', 'hash_multimap',)), + ('', ('hash_set', 'hash_multiset',)), + ('', ('slist',)), + ) + +_HEADERS_ACCEPTED_BUT_NOT_PROMOTED = { + # We can trust with reasonable confidence that map gives us pair<>, too. + 'pair<>': ('map', 'multimap', 'hash_map', 'hash_multimap') +} + +_RE_PATTERN_STRING = re.compile(r'\bstring\b') + +_re_pattern_algorithm_header = [] +for _template in ('copy', 'max', 'min', 'min_element', 'sort', 'swap', + 'transform'): + # Match max(..., ...), max(..., ...), but not foo->max, foo.max or + # type::max(). + _re_pattern_algorithm_header.append( + (re.compile(r'[^>.]\b' + _template + r'(<.*?>)?\([^\)]'), + _template, + '')) + +_re_pattern_templates = [] +for _header, _templates in _HEADERS_CONTAINING_TEMPLATES: + for _template in _templates: + _re_pattern_templates.append( + (re.compile(r'(\<|\b)' + _template + r'\s*\<'), + _template + '<>', + _header)) + + +def FilesBelongToSameModule(filename_cc, filename_h): + """Check if these two filenames belong to the same module. + + The concept of a 'module' here is a as follows: + foo.h, foo-inl.h, foo.cc, foo_test.cc and foo_unittest.cc belong to the + same 'module' if they are in the same directory. + some/path/public/xyzzy and some/path/internal/xyzzy are also considered + to belong to the same module here. + + If the filename_cc contains a longer path than the filename_h, for example, + '/absolute/path/to/base/sysinfo.cc', and this file would include + 'base/sysinfo.h', this function also produces the prefix needed to open the + header. This is used by the caller of this function to more robustly open the + header file. We don't have access to the real include paths in this context, + so we need this guesswork here. + + Known bugs: tools/base/bar.cc and base/bar.h belong to the same module + according to this implementation. Because of this, this function gives + some false positives. This should be sufficiently rare in practice. + + Args: + filename_cc: is the path for the .cc file + filename_h: is the path for the header path + + Returns: + Tuple with a bool and a string: + bool: True if filename_cc and filename_h belong to the same module. + string: the additional prefix needed to open the header file. + """ + + if not filename_cc.endswith('.cc'): + return (False, '') + filename_cc = filename_cc[:-len('.cc')] + if filename_cc.endswith('_unittest'): + filename_cc = filename_cc[:-len('_unittest')] + elif filename_cc.endswith('_test'): + filename_cc = filename_cc[:-len('_test')] + filename_cc = filename_cc.replace('/public/', '/') + filename_cc = filename_cc.replace('/internal/', '/') + + if not filename_h.endswith('.h'): + return (False, '') + filename_h = filename_h[:-len('.h')] + if filename_h.endswith('-inl'): + filename_h = filename_h[:-len('-inl')] + filename_h = filename_h.replace('/public/', '/') + filename_h = filename_h.replace('/internal/', '/') + + files_belong_to_same_module = filename_cc.endswith(filename_h) + common_path = '' + if files_belong_to_same_module: + common_path = filename_cc[:-len(filename_h)] + return files_belong_to_same_module, common_path + + +def UpdateIncludeState(filename, include_state, io=codecs): + """Fill up the include_state with new includes found from the file. + + Args: + filename: the name of the header to read. + include_state: an _IncludeState instance in which the headers are inserted. + io: The io factory to use to read the file. Provided for testability. + + Returns: + True if a header was succesfully added. False otherwise. + """ + headerfile = None + try: + headerfile = io.open(filename, 'r', 'utf8', 'replace') + except IOError: + return False + linenum = 0 + for line in headerfile: + linenum += 1 + clean_line = CleanseComments(line) + match = _RE_PATTERN_INCLUDE.search(clean_line) + if match: + include = match.group(2) + # The value formatting is cute, but not really used right now. + # What matters here is that the key is in include_state. + include_state.setdefault(include, '%s:%d' % (filename, linenum)) + return True + + +def CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error, + io=codecs): + """Reports for missing stl includes. + + This function will output warnings to make sure you are including the headers + necessary for the stl containers and functions that you use. We only give one + reason to include a header. For example, if you use both equal_to<> and + less<> in a .h file, only one (the latter in the file) of these will be + reported as a reason to include the . + + Args: + filename: The name of the current file. + clean_lines: A CleansedLines instance containing the file. + include_state: An _IncludeState instance. + error: The function to call with any errors found. + io: The IO factory to use to read the header file. Provided for unittest + injection. + """ + required = {} # A map of header name to linenumber and the template entity. + # Example of required: { '': (1219, 'less<>') } + + for linenum in xrange(clean_lines.NumLines()): + line = clean_lines.elided[linenum] + if not line or line[0] == '#': + continue + + # String is special -- it is a non-templatized type in STL. + if _RE_PATTERN_STRING.search(line): + required[''] = (linenum, 'string') + + for pattern, template, header in _re_pattern_algorithm_header: + if pattern.search(line): + required[header] = (linenum, template) + + # The following function is just a speed up, no semantics are changed. + if not '<' in line: # Reduces the cpu time usage by skipping lines. + continue + + for pattern, template, header in _re_pattern_templates: + if pattern.search(line): + required[header] = (linenum, template) + + # The policy is that if you #include something in foo.h you don't need to + # include it again in foo.cc. Here, we will look at possible includes. + # Let's copy the include_state so it is only messed up within this function. + include_state = include_state.copy() + + # Did we find the header for this file (if any) and succesfully load it? + header_found = False + + # Use the absolute path so that matching works properly. + abs_filename = os.path.abspath(filename) + + # For Emacs's flymake. + # If cpplint is invoked from Emacs's flymake, a temporary file is generated + # by flymake and that file name might end with '_flymake.cc'. In that case, + # restore original file name here so that the corresponding header file can be + # found. + # e.g. If the file name is 'foo_flymake.cc', we should search for 'foo.h' + # instead of 'foo_flymake.h' + emacs_flymake_suffix = '_flymake.cc' + if abs_filename.endswith(emacs_flymake_suffix): + abs_filename = abs_filename[:-len(emacs_flymake_suffix)] + '.cc' + + # include_state is modified during iteration, so we iterate over a copy of + # the keys. + for header in include_state.keys(): #NOLINT + (same_module, common_path) = FilesBelongToSameModule(abs_filename, header) + fullpath = common_path + header + if same_module and UpdateIncludeState(fullpath, include_state, io): + header_found = True + + # If we can't find the header file for a .cc, assume it's because we don't + # know where to look. In that case we'll give up as we're not sure they + # didn't include it in the .h file. + # TODO(unknown): Do a better job of finding .h files so we are confident that + # not having the .h file means there isn't one. + if filename.endswith('.cc') and not header_found: + return + + # All the lines have been processed, report the errors found. + for required_header_unstripped in required: + template = required[required_header_unstripped][1] + if template in _HEADERS_ACCEPTED_BUT_NOT_PROMOTED: + headers = _HEADERS_ACCEPTED_BUT_NOT_PROMOTED[template] + if [True for header in headers if header in include_state]: + continue + if required_header_unstripped.strip('<>"') not in include_state: + error(filename, required[required_header_unstripped][0], + 'build/include_what_you_use', 4, + 'Add #include ' + required_header_unstripped + ' for ' + template) + + +def ProcessLine(filename, file_extension, + clean_lines, line, include_state, function_state, + class_state, error): + """Processes a single line in the file. + + Args: + filename: Filename of the file that is being processed. + file_extension: The extension (dot not included) of the file. + clean_lines: An array of strings, each representing a line of the file, + with comments stripped. + line: Number of line being processed. + include_state: An _IncludeState instance in which the headers are inserted. + function_state: A _FunctionState instance which counts function lines, etc. + class_state: A _ClassState instance which maintains information about + the current stack of nested class declarations being parsed. + error: A callable to which errors are reported, which takes 4 arguments: + filename, line number, error level, and message + + """ + raw_lines = clean_lines.raw_lines + CheckForFunctionLengths(filename, clean_lines, line, function_state, error) + if Search(r'\bNOLINT\b', raw_lines[line]): # ignore nolint lines + return + CheckForMultilineCommentsAndStrings(filename, clean_lines, line, error) + CheckStyle(filename, clean_lines, line, file_extension, error) + CheckLanguage(filename, clean_lines, line, file_extension, include_state, + error) + CheckForNonStandardConstructs(filename, clean_lines, line, + class_state, error) + CheckPosixThreading(filename, clean_lines, line, error) + CheckInvalidIncrement(filename, clean_lines, line, error) + + +def ProcessFileData(filename, file_extension, lines, error): + """Performs lint checks and reports any errors to the given error function. + + Args: + filename: Filename of the file that is being processed. + file_extension: The extension (dot not included) of the file. + lines: An array of strings, each representing a line of the file, with the + last element being empty if the file is termined with a newline. + error: A callable to which errors are reported, which takes 4 arguments: + """ + lines = (['// marker so line numbers and indices both start at 1'] + lines + + ['// marker so line numbers end in a known way']) + + include_state = _IncludeState() + function_state = _FunctionState() + class_state = _ClassState() + + CheckForCopyright(filename, lines, error) + + if file_extension == 'h': + CheckForHeaderGuard(filename, lines, error) + + RemoveMultiLineComments(filename, lines, error) + clean_lines = CleansedLines(lines) + for line in xrange(clean_lines.NumLines()): + ProcessLine(filename, file_extension, clean_lines, line, + include_state, function_state, class_state, error) + class_state.CheckFinished(filename, error) + + CheckForIncludeWhatYouUse(filename, clean_lines, include_state, error) + + # We check here rather than inside ProcessLine so that we see raw + # lines rather than "cleaned" lines. + CheckForUnicodeReplacementCharacters(filename, lines, error) + + CheckForNewlineAtEOF(filename, lines, error) + + +def ProcessFile(filename, vlevel): + """Does google-lint on a single file. + + Args: + filename: The name of the file to parse. + + vlevel: The level of errors to report. Every error of confidence + >= verbose_level will be reported. 0 is a good default. + """ + + _SetVerboseLevel(vlevel) + + try: + # Support the UNIX convention of using "-" for stdin. Note that + # we are not opening the file with universal newline support + # (which codecs doesn't support anyway), so the resulting lines do + # contain trailing '\r' characters if we are reading a file that + # has CRLF endings. + # If after the split a trailing '\r' is present, it is removed + # below. If it is not expected to be present (i.e. os.linesep != + # '\r\n' as in Windows), a warning is issued below if this file + # is processed. + + if filename == '-': + lines = codecs.StreamReaderWriter(sys.stdin, + codecs.getreader('utf8'), + codecs.getwriter('utf8'), + 'replace').read().split('\n') + else: + lines = codecs.open(filename, 'r', 'utf8', 'replace').read().split('\n') + + carriage_return_found = False + # Remove trailing '\r'. + for linenum in range(len(lines)): + if lines[linenum].endswith('\r'): + lines[linenum] = lines[linenum].rstrip('\r') + carriage_return_found = True + + except IOError: + sys.stderr.write( + "Skipping input '%s': Can't open for reading\n" % filename) + return + + # Note, if no dot is found, this will give the entire filename as the ext. + file_extension = filename[filename.rfind('.') + 1:] + + # When reading from stdin, the extension is unknown, so no cpplint tests + # should rely on the extension. + if (filename != '-' and file_extension != 'cc' and file_extension != 'h' + and file_extension != 'cpp'): + sys.stderr.write('Ignoring %s; not a .cc or .h file\n' % filename) + else: + ProcessFileData(filename, file_extension, lines, Error) + if carriage_return_found and os.linesep != '\r\n': + # Use 0 for linenum since outputing only one error for potentially + # several lines. + Error(filename, 0, 'whitespace/newline', 1, + 'One or more unexpected \\r (^M) found;' + 'better to use only a \\n') + + sys.stderr.write('Done processing %s\n' % filename) + + +def PrintUsage(message): + """Prints a brief usage string and exits, optionally with an error message. + + Args: + message: The optional error message. + """ + sys.stderr.write(_USAGE) + if message: + sys.exit('\nFATAL ERROR: ' + message) + else: + sys.exit(1) + + +def PrintCategories(): + """Prints a list of all the error-categories used by error messages. + + These are the categories used to filter messages via --filter. + """ + sys.stderr.write(_ERROR_CATEGORIES) + sys.exit(0) + + +def ParseArguments(args): + """Parses the command line arguments. + + This may set the output format and verbosity level as side-effects. + + Args: + args: The command line arguments: + + Returns: + The list of filenames to lint. + """ + try: + (opts, filenames) = getopt.getopt(args, '', ['help', 'output=', 'verbose=', + 'counting=', + 'filter=']) + except getopt.GetoptError: + PrintUsage('Invalid arguments.') + + verbosity = _VerboseLevel() + output_format = _OutputFormat() + filters = '' + counting_style = '' + + for (opt, val) in opts: + if opt == '--help': + PrintUsage(None) + elif opt == '--output': + if not val in ('emacs', 'vs7'): + PrintUsage('The only allowed output formats are emacs and vs7.') + output_format = val + elif opt == '--verbose': + verbosity = int(val) + elif opt == '--filter': + filters = val + if not filters: + PrintCategories() + elif opt == '--counting': + if val not in ('total', 'toplevel', 'detailed'): + PrintUsage('Valid counting options are total, toplevel, and detailed') + counting_style = val + + if not filenames: + PrintUsage('No files were specified.') + + _SetOutputFormat(output_format) + _SetVerboseLevel(verbosity) + _SetFilters(filters) + _SetCountingStyle(counting_style) + + return filenames + + +def main(): + filenames = ParseArguments(sys.argv[1:]) + + # Change stderr to write with replacement characters so we don't die + # if we try to print something containing non-ASCII characters. + sys.stderr = codecs.StreamReaderWriter(sys.stderr, + codecs.getreader('utf8'), + codecs.getwriter('utf8'), + 'replace') + + _cpplint_state.ResetErrorCounts() + for filename in filenames: + ProcessFile(filename, _cpplint_state.verbose_level) + _cpplint_state.PrintErrorCounts() + + sys.exit(_cpplint_state.error_count > 0) + + +if __name__ == '__main__': + main() diff -r 000000000000 -r 582cbe817f2c src/Main/aimc.cc --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/Main/aimc.cc Fri Feb 12 12:31:23 2010 +0000 @@ -0,0 +1,28 @@ +// Copyright 2006-2010, Thomas Walters +// +// AIM-C: A C++ implementation of the Auditory Image Model +// http://www.acousticscale.org/AIMC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +#include +#include + +using std::vector; +using boost::scoped_ptr; + +int main () { + scoped_ptr > fred; + fred.reset(new vector); +} \ No newline at end of file diff -r 000000000000 -r 582cbe817f2c src/Modules/BMM/ModulePZFC.cc --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/Modules/BMM/ModulePZFC.cc Fri Feb 12 12:31:23 2010 +0000 @@ -0,0 +1,453 @@ +// Copyright 2008-2010, Thomas Walters +// +// AIM-C: A C++ implementation of the Auditory Image Model +// http://www.acousticscale.org/AIMC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +/*! \file + * \brief Dick Lyon's Pole-Zero Filter Cascade - implemented as an AIM-C + * module by Tom Walters from the AIM-MAT module based on Dick Lyon's code + */ + +/*! \author Thomas Walters + * \date created 2008/02/05 + * \version \$Id: ModulePZFC.cc 4 2010-02-03 18:44:58Z tcw $ + */ + +#include "Support/ERBTools.h" + +#include "Modules/BMM/ModulePZFC.h" + +namespace aimc { +ModulePZFC::ModulePZFC(Parameters *parameters) : Module(parameters) { + module_identifier_ = "pzfc"; + module_type_ = "bmm"; + module_description_ = "Pole-Zero Filter Cascade"; + module_version_ = "$Id: ModulePZFC.cc 4 2010-02-03 18:44:58Z tcw $"; + + // Get parameter values, setting default values where necessary + // Each parameter is set here only if it has not already been set elsewhere. + cf_max_ = parameters_->DefaultFloat("pzfc.highest_frequency", 6000.0f); + cf_min_ = parameters_->DefaultFloat("pzfc.lowest_frequency", 100.0f); + pole_damping_ = parameters_->DefaultFloat("pzfc.pole_damping", 0.12f); + zero_damping_ = parameters_->DefaultFloat("pzfc.zero_damping", 0.2f); + zero_factor_ = parameters_->DefaultFloat("pzfc.zero_factor", 1.4f); + step_factor_ = parameters_->DefaultFloat("pzfc.step_factor", 1.0f/3.0f); + bandwidth_over_cf_ = parameters_->DefaultFloat("pzfc.bandwidth_over_cf", + 0.11f); + min_bandwidth_hz_ = parameters_->DefaultFloat("pzfc.min_bandwidth_hz", + 27.0f); + agc_factor_ = parameters_->DefaultFloat("pzfc.agc_factor", 12.0f); + do_agc_step_ = parameters_->DefaultBool("pzfc.do_agc", true); + + detect_.resize(0); +} + +ModulePZFC::~ModulePZFC() { +} + +bool ModulePZFC::InitializeInternal(const SignalBank &input) { + // Make local convenience copies of some variables + sample_rate_ = input.sample_rate(); + buffer_length_ = input.buffer_length(); + channel_count_ = 0; + + // Prepare the coefficients and also the output SignalBank + if (!SetPZBankCoeffs()) + return false; + + // The output signal bank should be set up by now. + if (!output_.initialized()) + return false; + + // This initialises all buffers which can be modified by Process() + Reset(); + + return true; +} + +void ModulePZFC::Reset() { + // These buffers may be actively modified by the algorithm + agc_state_.clear(); + agc_state_.resize(channel_count_); + for (int i = 0; i < channel_count_; ++i) { + agc_state_[i].clear(); + agc_state_[i].resize(agc_stage_count_, 0.0f); + } + + state_1_.clear(); + state_1_.resize(channel_count_, 0.0f); + + state_2_.clear(); + state_2_.resize(channel_count_, 0.0f); + + previous_out_.clear(); + previous_out_.resize(channel_count_, 0.0f); + + pole_damps_mod_.clear(); + pole_damps_mod_.resize(channel_count_, 0.0f); + + inputs_.clear(); + inputs_.resize(channel_count_, 0.0f); + + // Init AGC + AGCDampStep(); + // pole_damps_mod_ and agc_state_ are now be initialized + + // Modify the pole dampings and AGC state slightly from their values in + // silence in case the input is abuptly loud. + for (int i = 0; i < channel_count_; ++i) { + pole_damps_mod_[i] += 0.05f; + for (int j = 0; j < agc_stage_count_; ++j) + agc_state_[i][j] += 0.05f; + } + + last_input_ = 0.0f; +} + +bool ModulePZFC::SetPZBankCoeffsERBFitted() { + float parameter_values[3 * 7] = { + // Filed, Nfit = 524, 11-3 parameters, PZFC, cwt 0, fit time 9915 sec + 1.14827, 0.00000, 0.00000, // % SumSqrErr= 10125.41 + 0.53571, -0.70128, 0.63246, // % RMSErr = 2.81586 + 0.76779, 0.00000, 0.00000, // % MeanErr = 0.00000 + // Inf 0.00000 0.00000 % RMSCost = NaN + 0.00000, 0.00000, 0.00000, + 6.00000, 0.00000, 0.00000, + 1.08869, -0.09470, 0.07844, + 10.56432, 2.52732, 1.86895 + // -3.45865 -1.31457 3.91779 % Kv + }; + + // Precalculate the number of channels required - this method is ugly but it + // was the quickest way of converting from MATLAB as the step factor between + // channels can vary quadratically with pole frequency... + + // Normalised maximum pole frequency + float pole_frequency = cf_max_ / sample_rate_ * (2.0f * M_PI); + + channel_count_ = 0; + while ((pole_frequency / (2.0f * M_PI)) * sample_rate_ > cf_min_) { + float frequency = pole_frequency / (2.0f * M_PI) * sample_rate_; + float f_dep = ERBTools::Freq2ERB(frequency) + / ERBTools::Freq2ERB(1000.0f) - 1.0f; + float bw = ERBTools::Freq2ERBw(pole_frequency + / (2.0f * M_PI) * sample_rate_); + float step_factor = 1.0f + / (parameter_values[4*3] + parameter_values[4 * 3 + 1] + * f_dep + parameter_values[4 * 3 + 2] * f_dep * f_dep); // 1/n2 + pole_frequency -= step_factor * (bw * (2.0f * M_PI) / sample_rate_); + channel_count_++; + } + + // Now the number of channels is known, various buffers for the filterbank + // coefficients can be initialised + pole_dampings_.clear(); + pole_dampings_.resize(channel_count_, 0.0f); + pole_frequencies_.clear(); + pole_frequencies_.resize(channel_count_, 0.0f); + + // Direct-form coefficients + za0_.clear(); + za0_.resize(channel_count_, 0.0f); + za1_.clear(); + za1_.resize(channel_count_, 0.0f); + za2_.clear(); + za2_.resize(channel_count_, 0.0f); + + // The output signal bank + output_.Initialize(channel_count_, buffer_length_, sample_rate_); + + // Reset the pole frequency to maximum + pole_frequency = cf_max_ / sample_rate_ * (2.0f * M_PI); + + for (int i = channel_count_ - 1; i > -1; --i) { + // Store the normalised pole frequncy + pole_frequencies_[i] = pole_frequency; + + // Calculate the real pole frequency from the normalised pole frequency + float frequency = pole_frequency / (2.0f * M_PI) * sample_rate_; + + // Store the real pole frequency as the 'centre frequency' of the filterbank + // channel + output_.set_centre_frequency(i, frequency); + + // From PZFC_Small_Signal_Params.m { From PZFC_Params.m { + float DpndF = ERBTools::Freq2ERB(frequency) + / ERBTools::Freq2ERB(1000.0f) - 1.0f; + + float p[8]; // Parameters (short name for ease of reading) + + // Use parameter_values to recover the parameter values for this frequency + for (int param = 0; param < 7; ++param) + p[param] = parameter_values[param * 3] + + parameter_values[param * 3 + 1] * DpndF + + parameter_values[param * 3 + 2] * DpndF * DpndF; + + // Calculate the final parameter + p[7] = p[1] * pow(10.0f, (p[2] / (p[1] * p[4])) * (p[6] - 60.0f) / 20.0f); + if (p[7] < 0.2f) + p[7] = 0.2f; + + // Nominal bandwidth at this frequency + float fERBw = ERBTools::Freq2ERBw(frequency); + + // Pole bandwidth + float fPBW = ((p[7] * fERBw * (2 * M_PI) / sample_rate_) / 2) + * pow(p[4], 0.5f); + + // Pole damping + float pole_damping = fPBW / sqrt(pow(pole_frequency, 2) + pow(fPBW, 2)); + + // Store the pole damping + pole_dampings_[i] = pole_damping; + + // Zero bandwidth + float fZBW = ((p[0] * p[5] * fERBw * (2 * M_PI) / sample_rate_) / 2) + * pow(p[4], 0.5f); + + // Zero frequency + float zero_frequency = p[5] * pole_frequency; + + if (zero_frequency > M_PI) + LOG_ERROR(_T("Warning: Zero frequency is above the Nyquist frequency " + "in ModulePZFC(), continuing anyway but results may not " + "be accurate.")); + + // Zero damping + float fZDamp = fZBW / sqrt(pow(zero_frequency, 2) + pow(fZBW, 2)); + + // Impulse-invariance mapping + float fZTheta = zero_frequency * sqrt(1.0f - pow(fZDamp, 2)); + float fZRho = exp(-fZDamp * zero_frequency); + + // Direct-form coefficients + float fA1 = -2.0f * fZRho * cos(fZTheta); + float fA2 = fZRho * fZRho; + + // Normalised to unity gain at DC + float fASum = 1.0f + fA1 + fA2; + za0_[i] = 1.0f / fASum; + za1_[i] = fA1 / fASum; + za2_[i] = fA2 / fASum; + + // Subtract step factor (1/n2) times current bandwidth from the pole + // frequency + pole_frequency -= ((1.0f / p[4]) + * (fERBw * (2.0f * M_PI) / sample_rate_)); + } +return true; +} + +bool ModulePZFC::SetPZBankCoeffs() { + /*! \todo Re-implement the alternative parameter settings + */ + if (!SetPZBankCoeffsERBFitted()) + return false; + + /*! \todo Make fMindamp and fMaxdamp user-settable? + */ + mindamp_ = 0.18f; + maxdamp_ = 0.4f; + + rmin_.resize(channel_count_); + rmax_.resize(channel_count_); + xmin_.resize(channel_count_); + xmax_.resize(channel_count_); + + for (int c = 0; c < channel_count_; ++c) { + // Calculate maximum and minimum damping options + rmin_[c] = exp(-mindamp_ * pole_frequencies_[c]); + rmax_[c] = exp(-maxdamp_ * pole_frequencies_[c]); + + xmin_[c] = rmin_[c] * cos(pole_frequencies_[c] + * pow((1-pow(mindamp_, 2)), 0.5f)); + xmax_[c] = rmax_[c] * cos(pole_frequencies_[c] + * pow((1-pow(maxdamp_, 2)), 0.5f)); + } + + // Set up AGC parameters + agc_stage_count_ = 4; + agc_epsilons_.resize(agc_stage_count_); + agc_epsilons_[0] = 0.0064f; + agc_epsilons_[1] = 0.0016f; + agc_epsilons_[2] = 0.0004f; + agc_epsilons_[3] = 0.0001f; + + agc_gains_.resize(agc_stage_count_); + agc_gains_[0] = 1.0f; + agc_gains_[1] = 1.4f; + agc_gains_[2] = 2.0f; + agc_gains_[3] = 2.8f; + + float mean_agc_gain = 0.0f; + for (int c = 0; c < agc_stage_count_; ++c) + mean_agc_gain += agc_gains_[c]; + mean_agc_gain /= static_cast(agc_stage_count_); + + for (int c = 0; c < agc_stage_count_; ++c) + agc_gains_[c] /= mean_agc_gain; + + return true; +} + +void ModulePZFC::AGCDampStep() { + if (detect_.size() == 0) { + // If detect_ is not initialised, it means that the AGC is not set up. + // Set up now. + /*! \todo Make a separate InitAGC function which does this. + */ + detect_.resize(channel_count_); + for (int c = 0; c < channel_count_; ++c) + detect_[c] = 1.0f; + + float fDetectZero = DetectFun(0.0f); + for (int c = 0; c < channel_count_; c++) + detect_[c] *= fDetectZero; + + for (int c = 0; c < channel_count_; c++) + for (int st = 0; st < agc_stage_count_; st++) + agc_state_[c][st] = (1.2f * detect_[c] * agc_gains_[st]); + } + + float fAGCEpsLeft = 0.3f; + float fAGCEpsRight = 0.3f; + + for (int c = channel_count_ - 1; c > -1; --c) { + for (int st = 0; st < agc_stage_count_; ++st) { + // This bounds checking is ugly and wasteful, and in an inner loop. + // If this algorithm is slow, this is why! + /*! \todo Proper non-ugly bounds checking in AGCDampStep() + */ + float fPrevAGCState; + float fCurrAGCState; + float fNextAGCState; + + if (c < channel_count_ - 1) + fPrevAGCState = agc_state_[c + 1][st]; + else + fPrevAGCState = agc_state_[c][st]; + + fCurrAGCState = agc_state_[c][st]; + + if (c > 0) + fNextAGCState = agc_state_[c - 1][st]; + else + fNextAGCState = agc_state_[c][st]; + + // Spatial smoothing + /*! \todo Something odd is going on here + * I think this line is not quite right. + */ + float agc_avg = fAGCEpsLeft * fPrevAGCState + + (1.0f - fAGCEpsLeft - fAGCEpsRight) * fCurrAGCState + + fAGCEpsRight * fNextAGCState; + // Temporal smoothing + agc_state_[c][st] = agc_avg * (1.0f - agc_epsilons_[st]) + + agc_epsilons_[st] * detect_[c] * agc_gains_[st]; + } + } + + float fOffset = 1.0f - agc_factor_ * DetectFun(0.0f); + + for (int i = 0; i < channel_count_; ++i) { + float fAGCStateMean = 0.0f; + for (int j = 0; j < agc_stage_count_; ++j) + fAGCStateMean += agc_state_[i][j]; + + fAGCStateMean /= static_cast(agc_stage_count_); + + pole_damps_mod_[i] = pole_dampings_[i] * + (fOffset + agc_factor_ * fAGCStateMean); + } +} + +float ModulePZFC::DetectFun(float fIN) { + if (fIN < 0.0f) + fIN = 0.0f; + float fDetect = Minimum(1.0f, fIN); + float fA = 0.25f; + return fA * fIN + (1.0f - fA) * (fDetect - pow(fDetect, 3) / 3.0f); +} + +inline float ModulePZFC::Minimum(float a, float b) { + if (a < b) + return a; + else + return b; +} + +void ModulePZFC::Process(const SignalBank& input) { + // Set the start time of the output buffer + output_.set_start_time(input.start_time()); + + for (int iSample = 0; iSample < input.buffer_length(); ++iSample) { + float fInput = input[0][iSample]; + + // Lowpass filter the input with a zero at PI + fInput = 0.5f * fInput + 0.5f * last_input_; + last_input_ = input[0][iSample]; + + inputs_[channel_count_ - 1] = fInput; + for (int c = 0; c < channel_count_ - 1; ++c) + inputs_[c] = previous_out_[c + 1]; + + // PZBankStep2 + // to save a bunch of divides + float damp_rate = 1.0f / (maxdamp_ - mindamp_); + + for (int c = channel_count_ - 1; c > -1; --c) { + float interp_factor = (pole_damps_mod_[c] + - mindamp_) * damp_rate; + + float x = xmin_[c] + (xmax_[c] - xmin_[c]) * interp_factor; + float r = rmin_[c] + (rmax_[c] - rmin_[c]) * interp_factor; + + // optional improvement to constellation adds a bit to r + float fd = pole_frequencies_[c] * pole_damps_mod_[c]; + // quadratic for small values, then linear + r = r + 0.25f * fd * Minimum(0.05f, fd); + + float zb1 = -2.0f * x; + float zb2 = r * r; + + /* canonic poles but with input provided where unity DC gain is assured + * (mean value of state is always equal to mean value of input) + */ + float new_state = inputs_[c] - (state_1_[c] - inputs_[c]) * zb1 + - (state_2_[c] - inputs_[c]) * zb2; + + // canonic zeros part as before: + float output = za0_[c] * new_state + za1_[c] * state_1_[c] + + za2_[c] * state_2_[c]; + + // cubic compression nonlinearity + output = output - 0.0001f * pow(output, 3); + + output_.set_sample(c, iSample, output); + detect_[c] = DetectFun(output); + state_2_[c] = state_1_[c]; + state_1_[c] = new_state; + } + + if (do_agc_step_) + AGCDampStep(); + + for (int c = 0; c < channel_count_; ++c) + previous_out_[c] = output_[c][iSample]; + } + PushOutput(); +} +} // namespace aimc diff -r 000000000000 -r 582cbe817f2c src/Modules/BMM/ModulePZFC.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/Modules/BMM/ModulePZFC.h Fri Feb 12 12:31:23 2010 +0000 @@ -0,0 +1,118 @@ +// Copyright 2008-2010, University of Cambridge +// +// AIM-C: A C++ implementation of the Auditory Image Model +// http://www.acousticscale.org/AIMC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! \file +//! \brief Dick Lyon's Pole-Zero Filter Cascade - implemented in C++ by Tom +//! Walters from the AIM-MAT module based on Dick Lyon's code. +//! +//! \author Thomas Walters +//! \date created 2008/02/05 +//! \version \$Id: ModulePZFC.h 2 2010-02-02 12:59:50Z tcw $ +//! +#ifndef _AIMC_MODULES_BMM_PZFC_H_ +#define _AIMC_MODULES_BMM_PZFC_H_ + +#include + +#include "Support/Module.h" +#include "Support/Parameters.h" +#include "Support/SignalBank.h" + +namespace aimc { +using std::vector; +class ModulePZFC : public Module { + public: + ModulePZFC(Parameters *pParam); + virtual ~ModulePZFC(); + + //! \brief Process a buffer + virtual void Process(const SignalBank &input); + + //! \brief Reset all internal state variables to their initial values + virtual void Reset(); + + private: + //! \brief Prepare the module + //! \param input Input SignalBank + //! \param output true on success false on failure + virtual bool InitializeInternal(const SignalBank &input); + + //! \brief Set the filterbank parameters according to a fit matrix from Unoki + //! and Lyon's fitting routine + bool SetPZBankCoeffsERBFitted(); + + //! \brief Sets the general filterbank coefficients + bool SetPZBankCoeffs(); + + //! \brief Automatic Gain Control + void AGCDampStep(); + + //! \brief Detector function - halfwave rectification etc. Used internally, + //! but not applied to the output. + float DetectFun(float fIN); + + //! \brief Minimum + inline float Minimum(float a, float b); + + int channel_count_; + int buffer_length_; + int agc_stage_count_; + float sample_rate_; + float last_input_; + + // Parameters + // User-settable scalars + float pole_damping_; + float zero_damping_; + float zero_factor_; + float step_factor_; + float bandwidth_over_cf_; + float min_bandwidth_hz_; + float agc_factor_; + float cf_max_; + float cf_min_; + float mindamp_; + float maxdamp_; + bool do_agc_step_; + + // Internal Buffers + // Initialised once + vector pole_dampings_; + vector agc_epsilons_; + vector agc_gains_; + vector pole_frequencies_; + vector za0_; + vector za1_; + vector za2_; + vector rmin_; + vector rmax_; + vector xmin_; + vector xmax_; + + // Modified by algorithm at each time step + vector detect_; + vector > agc_state_; + vector state_1_; + vector state_2_; + vector previous_out_; + vector pole_damps_mod_; + vector inputs_; +}; +} + +#endif // _AIMC_MODULES_BMM_PZFC_H_ diff -r 000000000000 -r 582cbe817f2c src/Modules/Features/ModuleGaussians.cc --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/Modules/Features/ModuleGaussians.cc Fri Feb 12 12:31:23 2010 +0000 @@ -0,0 +1,282 @@ +// Copyright 2008-2010, Thomas Walters +// +// AIM-C: A C++ implementation of the Auditory Image Model +// http://www.acousticscale.org/AIMC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +/*! \file + * \brief Gaussian features - based on MATLAB code by Christian Feldbauer + */ + +/*! + * \author Tom Walters + * \date created 2008/06/23 + * \version \$Id: ModuleGaussians.cc 2 2010-02-02 12:59:50Z tcw $ + */ + +#include + +#include "Modules/Features/ModuleGaussians.h" +#include "Support/Common.h" + +namespace aimc { +ModuleGaussians::ModuleGaussians(Parameters *pParam) +: Module(pParam) { + // Set module metadata + module_description_ = "Gaussian Fitting to SSI profile"; + module_identifier_ = "gaussians"; // unique identifier for the module + module_type_ = "features"; + module_version_ = "$Id: ModuleGaussians.cc 2 2010-02-02 12:59:50Z tcw $"; + + parameters_->SetDefault("features.gaussians.ncomp", "4"); + m_iParamNComp = parameters_->GetInt("features.gaussians.ncomp"); + + parameters_->SetDefault("features.gaussians.var", "115.0"); + m_fParamVar = parameters_->GetFloat("features.gaussians.var"); + + parameters_->SetDefault("features.gaussians.posterior_exp", "6.0"); + m_fParamPosteriorExp = + parameters_->GetFloat("features.gaussians.posterior_exp"); + + parameters_->SetDefault("features.gaussians.maxit", "250"); + m_iParamMaxIt = parameters_->GetInt("features.gaussians.maxit"); + + parameters_->SetDefault("features.gaussians.priors_converged", "1e-7"); + m_fParamPriorsConverged = + parameters_->GetInt("features.gaussians.priors_converged"); +} + +ModuleGaussians::~ModuleGaussians() { +} + +bool ModuleGaussians::InitializeInternal(const SignalBank &input) { + m_pA.resize(m_iParamNComp, 0.0f); + m_pMu.resize(m_iParamNComp, 0.0f); + + // Assuming the number of channels is greater than twice the number of + // Gaussian components, this is ok + if (input.channel_count() >= 2 * m_iParamNComp) { + output_.Initialize(1, m_iParamNComp, input.sample_rate()); + } else { + LOG_ERROR(_T("Too few channels in filterbank to produce sensible " + "Gaussian features. Either increase the number of filterbank" + " channels, or decrease the number of Gaussian components")); + return false; + } + + m_iNumChannels = input.channel_count(); + m_pSpectralProfile.resize(m_iNumChannels, 0.0f); + + return true; +} + +void ModuleGaussians::Reset() { + m_pSpectralProfile.clear(); + m_pSpectralProfile.resize(m_iNumChannels, 0.0f); +} + +void ModuleGaussians::Process(const SignalBank &input) { + int iAudCh = 0; + + // Calculate spectral profile + for (int iChannel = 0; + iChannel < input.channel_count(); + ++iChannel) { + m_pSpectralProfile[iChannel] = 0.0f; + for (int iSample = 0; + iSample < input.buffer_length(); + ++iSample) { + m_pSpectralProfile[iChannel] += input[iChannel][iSample]; + } + } + + for (int iChannel = 0; + iChannel < input.channel_count(); + ++iChannel) { + m_pSpectralProfile[iChannel] = pow(m_pSpectralProfile[iChannel], 0.8); + } + + float spectral_profile_sum = 0.0f; + for (int i = 0; i < input.channel_count(); ++i) { + spectral_profile_sum += m_pSpectralProfile[i]; + } + + RubberGMMCore(2, true); + + float fMean1 = m_pMu[0]; + float fMean2 = m_pMu[1]; + + float fA1 = 0.05 * m_pA[0]; + float fA2 = 1.0 - 0.25 * m_pA[1]; + + float fGradient = (fMean2 - fMean1) / (fA2 - fA1); + float fIntercept = fMean2 - fGradient * fA2; + + for (int i = 0; i < m_iParamNComp; ++i) { + m_pMu[i] = ((float)i / (float)m_iParamNComp - 1.0f) + * -fGradient + fIntercept; + } + + for (int i = 0; i < m_iParamNComp; ++i) { + m_pA[i] = 1.0f / (float)m_iParamNComp; + } + + RubberGMMCore(m_iParamNComp, false); + + for (int i = 0; i < m_iParamNComp - 1; ++i) { + if (!isnan(m_pA[i])) { + output_.set_sample(i, 0, m_pA[i]); + } else { + output_.set_sample(i, 0, 0.0f); + } + } + /*for (int i = m_iParamNComp; i < m_iParamNComp * 2; ++i) { + m_pOutputData->getSignal(i)->setSample(iAudCh, 0, m_pMu[i-m_iParamNComp]); + }*/ + double logsum = log(spectral_profile_sum); + if (!isinf(logsum)) { + output_.set_sample(m_iParamNComp - 1, 0, logsum); + } else { + output_.set_sample(m_iParamNComp - 1, 0, -1000.0); + } + PushOutput(); +} + +bool ModuleGaussians::RubberGMMCore(int iNComponents, bool bDoInit) { + int iSizeX = m_iNumChannels; + + // Normalise the spectral profile + float fSpectralProfileTotal = 0.0f; + for (int iCount = 0; iCount < iSizeX; iCount++) { + fSpectralProfileTotal += m_pSpectralProfile[iCount]; + } + for (int iCount = 0; iCount < iSizeX; iCount++) { + m_pSpectralProfile[iCount] /= fSpectralProfileTotal; + } + + if (bDoInit) { + // Uniformly spaced components + float dd = (iSizeX - 1.0f) / iNComponents; + for (int i = 0; i < iNComponents; i++) { + m_pMu[i] = dd / 2.0f + (i * dd); + m_pA[i] = 1.0f / iNComponents; + } + } + + vector pA_old; + pA_old.resize(iNComponents); + vector pP_mod_X; + pP_mod_X.resize(iSizeX); + vector pP_comp; + pP_comp.resize(iSizeX * iNComponents); + + for (int iIteration = 0; iIteration < m_iParamMaxIt; iIteration++) { + // (re)calculate posteriors (component probability given observation) + // denominator: the model density at all observation points X + for (int i = 0; i < iSizeX; ++i) { + pP_mod_X[i] = 0.0f; + } + + for (int i = 0; i < iNComponents; i++) { + for (int iCount = 0; iCount < iSizeX; iCount++) { + pP_mod_X[iCount] += 1.0f / sqrt(2.0f * M_PI * m_fParamVar) + * exp((-0.5f) * pow(((float)iCount-m_pMu[i]), 2) + / m_fParamVar) * m_pA[i]; + } + } + + for (int i = 0; i < iSizeX * iNComponents; ++i) { + pP_comp[i] = 0.0f; + } + + for (int i = 0; i < iNComponents; i++) { + for (int iCount = 0; iCount < iSizeX; iCount++) { + pP_comp[iCount + i * iSizeX] = + 1.0f / sqrt(2.0f * M_PI * m_fParamVar) + * exp((-0.5f) * pow(((float)iCount - m_pMu[i]), 2) / m_fParamVar); + pP_comp[iCount + i * iSizeX] = + pP_comp[iCount + i * iSizeX] * m_pA[i] / pP_mod_X[iCount]; + } + } + + for (int iCount = 0; iCount < iSizeX; ++iCount) { + float fSum = 0.0f; + for (int i = 0; i < iNComponents; ++i) { + pP_comp[iCount+i*iSizeX] = pow(pP_comp[iCount + i * iSizeX], + m_fParamPosteriorExp); // expansion + fSum += pP_comp[iCount+i*iSizeX]; + } + for (int i = 0; i < iNComponents; ++i) + pP_comp[iCount+i*iSizeX] = pP_comp[iCount + i * iSizeX] / fSum; + // renormalisation + } + + for (int i = 0; i < iNComponents; ++i) { + pA_old[i] = m_pA[i]; + m_pA[i] = 0.0f; + for (int iCount = 0; iCount < iSizeX; ++iCount) { + m_pA[i] += pP_comp[iCount + i * iSizeX] * m_pSpectralProfile[iCount]; + } + } + + // finish when already converged + float fPrdist = 0.0f; + for (int i = 0; i < iNComponents; ++i) { + fPrdist += pow((m_pA[i] - pA_old[i]), 2); + } + fPrdist /= iNComponents; + + if (fPrdist < m_fParamPriorsConverged) { + LOG_INFO("Converged!"); + break; + } + + // update means (positions) + for (int i = 0 ; i < iNComponents; ++i) { + float mu_old = m_pMu[i]; + if (m_pA[i] > 0.0f) { + m_pMu[i] = 0.0f; + for (int iCount = 0; iCount < iSizeX; ++iCount) { + m_pMu[i] += m_pSpectralProfile[iCount] + * pP_comp[iCount + i * iSizeX] * (float)iCount; + } + m_pMu[i] /= m_pA[i]; + if (isnan(m_pMu[i])) { + m_pMu[i] = mu_old; + } + } + } + } // loop over iterations + + // Ensure they are sorted, using a really simple bubblesort + bool bSorted = false; + while (!bSorted) { + bSorted = true; + for (int i = 0; i < iNComponents - 1; ++i) { + if (m_pMu[i] > m_pMu[i + 1]) { + float fTemp = m_pMu[i]; + m_pMu[i] = m_pMu[i + 1]; + m_pMu[i + 1] = fTemp; + fTemp = m_pA[i]; + m_pA[i] = m_pA[i + 1]; + m_pA[i + 1] = fTemp; + bSorted = false; + } + } + } + return true; +} +} //namespace aimc + diff -r 000000000000 -r 582cbe817f2c src/Modules/Features/ModuleGaussians.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/Modules/Features/ModuleGaussians.h Fri Feb 12 12:31:23 2010 +0000 @@ -0,0 +1,99 @@ +// Copyright 2008-2010, University of Cambridge +// +// AIM-C: A C++ implementation of the Auditory Image Model +// http://www.acousticscale.org/AIMC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +/*! \file ModuleGaussians.h + * \brief Gaussian features + */ + +/*! \author Tom Walters + * \date created 2008/06/23 + * \version \$Id: ModuleGaussians.h 2 2010-02-02 12:59:50Z tcw $ + */ + +#ifndef _AIMC_MODULE_FEATURES_GAUSSIANS_H_ +#define _AIMC_MODULE_FEATURES_GAUSSIANS_H_ + +#include + +#include "Support/Module.h" +#include "Support/Parameters.h" +#include "Support/SignalBank.h" + +/*! \class ModuleGaussians "Modules/Features/ModuleGaussians.h" + * \brief + */ +namespace aimc { +using std::vector; +class ModuleGaussians : public Module +{ + public: + ModuleGaussians(Parameters *pParam); + virtual ~ModuleGaussians(); + + //! \brief Process a buffer + virtual void Process(const SignalBank &input); + + //! \brief Reset the internal state of the module + void Reset(); + + private: + /*! \brief Prepare the module + * \param input Input signal + * \param output true on success false on failure + */ + virtual bool InitializeInternal(const SignalBank &input); + + bool RubberGMMCore(int iNumComponents, bool bDoInit); + + /*! \brief Number of Gaussian Components + */ + int m_iParamNComp; + + /*! \brief Constant variance of Gaussians + */ + float m_fParamVar; + + /*! \brief posterior probability expansion exponent + */ + float m_fParamPosteriorExp; + + /*! \brief Maximum Number of iterations + */ + unsigned int m_iParamMaxIt; + + /*! \brief convergence criterion + */ + float m_fParamPriorsConverged; + + /*! \brief The amplitudes of the components (priors) + */ + vector m_pA; + + /*! \brief The means of the components (priors) + */ + vector m_pMu; + + /*! \brief The spectral profile of the incoming buffer + */ + vector m_pSpectralProfile; + + int m_iNumChannels; +}; +} // namespace aimc + +#endif // _AIMC_MODULE_FEATURES_GAUSSIANS_H_ diff -r 000000000000 -r 582cbe817f2c src/Modules/NAP/ModuleHCL.cc --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/Modules/NAP/ModuleHCL.cc Fri Feb 12 12:31:23 2010 +0000 @@ -0,0 +1,110 @@ +// Copyright 2007-2010, Thomas Walters +// +// AIM-C: A C++ implementation of the Auditory Image Model +// http://www.acousticscale.org/AIMC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +/*! + * \file + * \brief Halfwave rectification, compression and lowpass filtering. + * + * \author Tom Walters + * \date created 2007/03/07 + * \version \$Id: ModuleHCL.cc 4 2010-02-03 18:44:58Z tcw $ + */ + +#include + +#include "Modules/NAP/ModuleHCL.h" + +namespace aimc { +ModuleHCL::ModuleHCL(Parameters *parameters) : Module(parameters) { + module_identifier_ = "hcl"; + module_type_ = "nap"; + module_description_ = "Halfwave rectification, compression " + "and lowpass filtering"; + module_version_ = "$Id: ModuleHCL.cc 4 2010-02-03 18:44:58Z tcw $"; + + parameters_->SetDefault("nap.do_lowpass", "false"); + parameters_->SetDefault("nap.do_log_compression", "false"); + parameters_->SetDefault("nap.lowpass_cutoff", "1200.0"); + parameters_->SetDefault("nap.lowpass_order", "2"); + + do_lowpass_ = parameters_->GetBool("nap.do_lowpass"); + do_log_ = parameters_->GetBool("nap.do_log_compression"); + lowpass_cutoff_ = parameters_->GetFloat("nap.lowpass_cutoff"); + lowpass_order_ = parameters_->GetInt("nap.lowpass_order"); +} + +ModuleHCL::~ModuleHCL() { +} + +bool ModuleHCL::InitializeInternal(const SignalBank &input) { + time_constant_ = 1.0f / (2.0f * M_PI * lowpass_cutoff_); + channel_count_ = input.channel_count(); + output_.Initialize(input); + Reset(); + return true; +} + +void ModuleHCL::Reset() { + xn_ = 0.0f; + yn_ = 0.0f; + yns_.clear(); + yns_.resize(channel_count_); + for (int c = 0; c < channel_count_; ++c) { + yns_[c].resize(lowpass_order_, 0.0f); + } +} + +/* With do_log, the signal is first scaled up so that values <1.0 become + * negligible. This just rescales the sample values to fill the range of a + * 16-bit signed integer, then we lose the bottom bit of resolution. If the + * signal was sampled at 16-bit resolution, there shouldn't be anything to + * speak of there anyway. If it was sampled using a higher resolution, then + * some data will be discarded. + */ +void ModuleHCL::Process(const SignalBank &input) { + output_.set_start_time(input.start_time()); + for (int c = 0; c < input.channel_count(); ++c) { + for (int i = 0; i < input.buffer_length(); ++i) { + if (input[c][i] < 0.0f) { + output_.set_sample(c, i, 0.0f); + } else { + float s = input[c][i]; + if (do_log_) { + s *= pow(2.0f,15); + if (s < 1.0f) s = 1.0f; + s = 20.0f * log10(s); + } + output_.set_sample(c, i, s); + } + } + if (do_lowpass_) { + float b = exp( -1.0f / (input.sample_rate() * time_constant_)); + float gain = 1.0f / (1.0f - b); + for (int j = 0; j < lowpass_order_; j++) { + for (int k = 0; k < output_.buffer_length(); ++k) { + xn_ = output_[c][k]; + yn_ = xn_ + b * yns_[c][j]; + yns_[c][j] = yn_; + output_.set_sample(c, k, yn_ / gain); + } + } + } + } + PushOutput(); +} +} // namespace aimc diff -r 000000000000 -r 582cbe817f2c src/Modules/NAP/ModuleHCL.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/Modules/NAP/ModuleHCL.h Fri Feb 12 12:31:23 2010 +0000 @@ -0,0 +1,79 @@ +// Copyright 2007-2010, University of Cambridge +// +// AIM-C: A C++ implementation of the Auditory Image Model +// http://www.acousticscale.org/AIMC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +/*! + * \file + * \brief Halfwave rectification, compression and lowpass filtering + */ + +/* \author Tom Walters + * \date created 2007/03/07 + * \version \$Id: ModuleHCL.h 4 2010-02-03 18:44:58Z tcw $ + */ + +#ifndef _AIMC_MODULE_NAP_HCL_H_ +#define _AIMC_MODULE_NAP_HCL_H_ + +#include + +#include "Support/Module.h" +#include "Support/SignalBank.h" + +namespace aimc { +using std::vector; +class ModuleHCL : public Module { + public: + ModuleHCL(Parameters *parameters); + virtual ~ModuleHCL(); + + virtual void Process(const SignalBank &input); + virtual void Reset(); + + private: + /*! \brief Prepare the module + * \param input Input signal bank + * \param output true on success false on failure + */ + virtual bool InitializeInternal(const SignalBank &input); + + //! \brief Do lowpass filtering? + bool do_lowpass_; + + //! \brief Do log compression? + bool do_log_; + + //! \brief Cutoff frequency for lowpass filter + float lowpass_cutoff_; + + //! \brief Order of Lowpass Filter + int lowpass_order_; + + //! \brief Internal record of the number of channels in the input + int channel_count_; + + //! \brief Time constant corresponsing to the lowpass filter cutoff freqency + float time_constant_; + + //! \brief Lowpass filter state variables + float xn_; + float yn_; + vector > yns_; +}; +} // namespace aimc + +#endif // _AIMC_MODULE_NAP_HCL_H_ diff -r 000000000000 -r 582cbe817f2c src/Modules/SAI/ModuleSAI.cc --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/Modules/SAI/ModuleSAI.cc Fri Feb 12 12:31:23 2010 +0000 @@ -0,0 +1,261 @@ +// Copyright 2006-2010, Thomas Walters +// +// AIM-C: A C++ implementation of the Auditory Image Model +// http://www.acousticscale.org/AIMC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +/*! \file + * \brief SAI module + */ + +/* + * \author Thomas Walters + * \date created 2007/08/29 + * \version \$Id: ModuleSAI.cc 4 2010-02-03 18:44:58Z tcw $ + */ +#include + +#include "Modules/SAI/ModuleSAI.h" + +ModuleSAI::ModuleSAI(Parameters *parameters) : Module(parameters) { + module_description_ = "Stabilised auditory image"; + module_name_ = "sai2007"; + module_type_ = "sai"; + module_id_ = "$Id: ModuleSAI.cc 4 2010-02-03 18:44:58Z tcw $"; + + min_delay_ms_ = parameters_->DefaultFloat("sai.min_delay_ms", 0.0f); + max_delay_ms_ = parameters_->DefaultFloat("sai.max_delay_ms", 35.0f); + strobe_weight_alpha_ = parameters_->DefaultFloat("sai.strobe_weight_alpha", + 0.5f); + buffer_memory_decay_ = parameters_->DefaultFloat("sai.buffer_memory_decay", + 0.03f); + frame_period_ms_ = parameters_->DefaultFloat("sai.frame_period_ms", 20.0f); + + min_strobe_delay_index_ = 0; + max_strobe_delay_index_ = 0; + sai_decay_factor_ = 0.0f; + max_concurrent_strobes_ = 0; + output_frame_period_ms_ = 0.0f; + last_output_frame_time_ms_ = 0.0f; +} + +bool ModuleSAI::InitializeInternal(const SignalBank &input) { + // The SAI output bank must be as long as the SAI's Maximum delay. + // One sample is added to the SAI buffer length to account for the + // zero-lag point + int sai_buffer_length = 1 + Round(input.sample_rate() * max_delay_ms_); + + // Make an output SignalBank with the same number of channels and centre + // frequencies as the input, but with a different buffer length + if (!output_.Initialize(input.channel_count(), + sai_buffer_length, + input.sample_rate());) { + LOG_ERROR("Failed to create output buffer in SAI module"); + return false; + } + for (int i = 0; i < input.channel_count(); ++i ) { + output_.set_centre_frequency(i, input.get_centre_frequency(i)); + } + + // sai_temp_ will be initialized to zero + if (!sai_temp_.Initialize(output_)) { + LOG_ERROR("Failed to create temporary buffer in SAI module"); + return false; + } + + last_output_time_ms_ = 0.0f; + + frame_period_samples_ = Round(input.sample_rate() + * frame_period_ms_ / 1000.0f); + + min_strobe_delay_idx_ = Round(input.sample_rate() * min_delay_ms_ / 1000.0f); + max_strobe_delay_idx_ = Round(input.sample_rate() * max_delay_ms_ / 1000.0f); + + // Make sure we don't go past the output buffer's upper bound + if (max_strobe_delay_idx_ > output_.buffer_length())) + max_strobe_delay_idx_ = output_.buffer_length(); + + // Define decay factor from time since last sample (see ti2003) + sai_decay_factor_ = pow(0.5f, 1.0f / (buffer_memory_decay_ + * input.sample_rate())); + + // Maximum strobes that can be active at the same time within maxdelay. + //! \todo Choose this value in a more principled way + max_concurrent_strobes_ = Round(1000.0f * max_delay_ * 5); + + // Precompute strobe weights + strobe_weights_.resize(max_concurrent_strobes_); + for (int n = 0; n < max_concurrent_strobes_; ++n) { + strobe_weights_[n] = pow(1.0f / (n + 1)), strobe_weight_alpha_); + } + + // Active Strobes + active_strobes_.Resize(input.channel_count()); + for (int i = 0; i < input.channel_count(); ++i) { + active_strobes_[i].Create(max_concurrent_strobes_); + } + next_strobes_.resize(input.channel_count(), 0); + + return true; +} + +void ModuleSAI::Reset() { +} + +void ModuleSAI::Process(const SignalBank &input) { + int s; + int c; + int output_buffer_length = output_.buffer_length(); + + // Reset the next strobe times + next_strobes_.clear(); + next_strobes_.resize(output_.channel_count(), 0); + + // Offset the times on the strobes from the previous buffer + for (c = 0; c < input.channel_count(), ++c) { + active_strobes_[c].shiftStrobes(input.buffer_length()); + } + + // Make sure only start time is transferred to the output + output_.set_start_time(input.start_time()); + + // Loop over samples to make the SAI + for (s = 0; s < input_buffer_length; ++s) { + float decay_factor = pow(sai_decay_factor_, fire_counter_); + // Loop over channels + for (c = 0; c < input.channel_count(); ++c) { + // Local convenience variables + StrobeList &active_strobes = active_strobes_[c]; + float centre_frequency = input.get_centre_frequency(c); + int next_strobe = next_strobes_[c]; + + // 1. Update strobes + // If we are up to or beyond the next strobe... + if (next_strobe < input.strobe_count(c)) { + if (s == pSigIn->getStrobe(iNextStrobe)) { + //A new strobe has arrived + // if there aren't already too many strobes active... + if ((active_strobes.getStrobeCount() + 1) < max_concurrent_strobes_) { + // ...add the active strobe to the list of current strobes + // calculate the strobe weight + float weight = 1.0f; + if (active_strobes.getStrobeCount() > 0) { + int last_strobe = active_strobes.getTime( + active_strobes.getStrobeCount()); + + // If the strobe occured within 10 impulse-response + // cycles of the previous strobe, then lower its weight + weight = (s - iLastStrobe) / input.sample_rate() + * centre_frequency / 10.0f; + if (weight > 1.0f) + weight = 1.0f; + } + pActiveStrobes->addStrobe(iCurrentSample, weight); + iNextStrobe++; + } else { + // We have a problem + aimASSERT(0); + } + + // 2. Having updated the strobes, we now need to update the + // strobe weights + float total_strobe_weight = 0.0f; + for (int si = 1; si <= pActiveStrobes->getStrobeCount(); ++si) { + total_strobe_weight += (pActiveStrobes->getWeight(si) + * m_pStrobeWeights[pActiveStrobes->getStrobeCount() - si]); + } + for (int si = 1; si <= pActiveStrobes->getStrobeCount(); ++si) { + active_strobes.setWorkingWeight(si,(active_strobes.getWeight(si) + * strobe_weights_[active_strobes.getStrobeCount() - si]) + / total_strobe_weight); + } + } + } + + // remove inactive strobes... + while (pActiveStrobes->getStrobeCount() > 0) { + // Get the time of the first strobe (ordering of strobes is + // from one, not zero) + int iStrobeTime = pActiveStrobes->getTime(1); + int iDelay = iCurrentSample - iStrobeTime; + // ... do we now need to remove this strobe? + if (iDelay > m_maxStrobeDelayIdx) + pActiveStrobes->deleteFirstStrobe(); + else + break; + // Since the strobes are ordered, we don't need to go + // beyond the first still-active strobe + } + + // 3. Loop over active strobes + for (int si = 1; si <= pActiveStrobes->getStrobeCount(); si++) { + // 3.1 Add effect of active strobe at correct place in the SAI buffer + // Calculate the time from the strobe event to 'now': iDelay + int iStrobeTime = pActiveStrobes->getTime(si); + int iDelay = iCurrentSample - iStrobeTime; + + // If the delay is greater than the (user-set) + // minimum strobe delay, the strobe can be used + if (iDelay >= m_minStrobeDelayIdx && iDelay < m_maxStrobeDelayIdx) { + // The value at be added to the SAI + float sig = pSigIn->getSample(iCurrentSample, audCh); + + // Weight the sample correctly + sig *= pActiveStrobes->getWorkingWeight(si); + + // Adjust the weight acording to the number of samples until the + // next output frame + sig *= fDecayFactor; + + // Update the temporary SAI buffer + pSigOut->setSample(iDelay, audCh, + pSigOut->getSample(iDelay, audCh)+sig); + } + } + + m_pNextStrobes[bankCh]=iNextStrobe; + + } // End loop over channels + + + //Check to see if we need to output an SAI frame this sample + if (m_iFireCounter-- == 0) { + // Decay the SAI by the correct amount and add the current output frame + float decay = pow(sai_decay_factor_, fire_period_samples_); + + for (c = 0; c < input.channel_count(); ++c) { + for (int i = 0; i < output_.buffer_length(); ++i) { + output_.set_sample(c, i, sai_temp_[c][i] + output_[c][i] * decay); + } + } + + // Zero the temporary signal + for (int ch = 0; ch < sai_temp_.channel_count(); ++ch) { + for (int i = 0; i < sai_temp_.buffer_length(); ++i) { + sai_temp_.set_sample(ch, i, 0.0f); + } + } + + m_iFireCounter=m_iFirePeriodSamples-1; + + // Make sure the start time is transferred to the output + m_pOutputData->setStartTime(m_pInputData->getSignal(0)->getStartTime()+(SIGNAL_SAMPLE)((float)iCurrentSample*1000.0f/(float)m_pInputData->getSamplerate())); + PushOutput(); + } + } // End loop over samples +} + +ModuleSAI::~ModuleSAI() { +} diff -r 000000000000 -r 582cbe817f2c src/Modules/SAI/ModuleSAI.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/Modules/SAI/ModuleSAI.h Fri Feb 12 12:31:23 2010 +0000 @@ -0,0 +1,98 @@ +// Copyright 2006-2010, Thomas Walters +// +// AIM-C: A C++ implementation of the Auditory Image Model +// http://www.acousticscale.org/AIMC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +/*! \file + * \brief SAI module + */ + +/*! \author Thomas Walters + * \date created 2007/08/29 + * \version \$Id: ModuleSAI.h 4 2010-02-03 18:44:58Z tcw $ + */ +#ifndef _AIMC_MODULE_SAI_H_ +#define _AIMC_MODULE_SAI_H_ + +#include + +#include "Support/Module.h" +#include "Support/SignalBank.h" +#include "Support/StrobeList.h" + +namespace aimc { +using std::vector; +class ModuleSAI : public Module { + public: + ModuleSAI(Parameters *parameters); + virtual ~ModuleSAI(); + void Process(const SignalBank &input); + void Reset(); + + private: + /*! \brief Prepare the module + * \param input Input signal bank + * \param output true on success false on failure + */ + bool InitializeInternal(const SignalBank &input); + + //! \brief Temporary buffer for constructing the current SAI frame + SignalBank sai_temp_; + + //! \brief List of strobes for each channel + vector active_strobes_; + + //! \brief Buffer decay parameter + float buffer_memory_decay_; + + //! \brief Sample index of minimum strobe delay + int min_strobe_delay_idx_; + + //! \brief Sample index of maximum strobe delay + int max_strobe_delay_idx_; + + //! \brief Factor with which the SAI should be decayed + float sai_decay_factor_; + + //! \brief Precomputed 1/n^alpha values for strobe weighting + vector strobe_weights_; + + //! \brief Next Strobe for each channels + vector next_strobes_; + + float strobe_weight_alpha_; + + /*! \brief The maximum number strobes that can be active at the same time. + * + * A strobe lasts for strobe.maxdelay, there can only be a certain number + * of strobes active at the same time, that's this value. It's used for + * allocating memory buffers, like m_pUnfinishedStrobeCount and + * m_pStrobeWeights. + */ + int max_concurrent_strobes_; + + int fire_period_samples_; + int fire_counter_; + + //! \brief Period in milliseconds between output frames + float output_frame_period_ms_; + + //! \brief Time of the last frame output + float last_output_frame_time_ms_; +}; +} // namespace aimc + +#endif // _AIMC_MODULE_SAI_H_ diff -r 000000000000 -r 582cbe817f2c src/Support/Common.cc --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/Support/Common.cc Fri Feb 12 12:31:23 2010 +0000 @@ -0,0 +1,38 @@ +// Copyright 2006-2010, Thomas Walters, Willem van Engen +// +// AIM-C: A C++ implementation of the Auditory Image Model +// http://www.acousticscale.org/AIMC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +#include "Support/Common.h" + +namespace aimc { +void LOG_ERROR(const char *sFmt, ...) { + va_list args; + va_start(args, sFmt); + vfprintf(stderr, sFmt, args); + fprintf(stderr, "\n"); + va_end(args); +} + +void LOG_INFO(const char *sFmt, ...) { + va_list args; + va_start(args, sFmt); + // Just print message to console (will be lost on windows with gui) + vprintf(sFmt, args); + printf("\n"); + va_end(args); +} +} // namespace aimc diff -r 000000000000 -r 582cbe817f2c src/Support/Common.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/Support/Common.h Fri Feb 12 12:31:23 2010 +0000 @@ -0,0 +1,101 @@ +// Copyright 2006-2010, Thomas Walters, Willem van Engen +// +// AIM-C: A C++ implementation of the Auditory Image Model +// http://www.acousticscale.org/AIMC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +/*! \file + * \brief Common includes for all AIM-C + */ + +/*! \author: Thomas Walters + * \author: Willem van Engen + * \date 2010/01/30 + * \version \$Id: Common.h 1 2010-02-02 11:04:50Z tcw $ + */ + +#ifndef _AIMC_SUPPORT_COMMON_H_ +#define _AIMC_SUPPORT_COMMON_H_ + +#include +#include +#include + +// A macro to disallow the copy constructor and operator= functions +// This should be used in the private: declarations for a class +#define DISALLOW_COPY_AND_ASSIGN(TypeName) \ + TypeName(const TypeName&); \ + void operator=(const TypeName&) + +#if !defined(_T) +# ifdef _UNICODE +# define _T(x) L ## x +# else +# define _T(x) x +# endif +#endif + +/*! \brief C++ delete if != NULL + * + * This was used so often, that is was moved to a macro. + */ +#define DELETE_IF_NONNULL(x) { \ + if ( (x) ) { \ + delete (x); \ + (x) = NULL; \ + } \ +} + +/*! \brief C++ delete[] if != NULL + * + * This was used so often, that is was moved to a macro. + */ +#define DELETE_ARRAY_IF_NONNULL(x) { \ + if ( (x) ) { \ + delete[] (x); \ + (x) = NULL; \ + } \ +} + +/*! \brief C free if != NULL + * + * This was used so often, that is was moved to a macro. + */ +#define FREE_IF_NONNULL(x) { \ + if ( (x) ) { \ + free(x); \ + (x) = NULL; \ + } \ +} + +#ifdef DEBUG +# define AIM_VERIFY(x) AIM_ASSERT(x) +# define AIM_ASSERT(x) { \ + if (!(x)) { \ + LOG_ERROR("Assertion failed.\n"); \ + *((char*)0) = 0; \ + } \ + } +#else +# define AIM_VERIFY(x) {x;} +# define AIM_ASSERT(...) +#endif + +namespace aimc { +void LOG_ERROR(const char *sFmt, ...); +void LOG_INFO(const char *sFmt, ...); +} // namespace aimc + +#endif // _AIMC_SUPPORT_COMMON_H_ diff -r 000000000000 -r 582cbe817f2c src/Support/ConvertUTF.c --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/Support/ConvertUTF.c Fri Feb 12 12:31:23 2010 +0000 @@ -0,0 +1,539 @@ +/* + * Copyright 2001-2004 Unicode, Inc. + * + * Disclaimer + * + * This source code is provided as is by Unicode, Inc. No claims are + * made as to fitness for any particular purpose. No warranties of any + * kind are expressed or implied. The recipient agrees to determine + * applicability of information provided. If this file has been + * purchased on magnetic or optical media from Unicode, Inc., the + * sole remedy for any claim will be exchange of defective media + * within 90 days of receipt. + * + * Limitations on Rights to Redistribute This Code + * + * Unicode, Inc. hereby grants the right to freely use the information + * supplied in this file in the creation of products supporting the + * Unicode Standard, and to make copies of this file in any form + * for internal or external distribution as long as this notice + * remains attached. + */ + +/* --------------------------------------------------------------------- + + Conversions between UTF32, UTF-16, and UTF-8. Source code file. + Author: Mark E. Davis, 1994. + Rev History: Rick McGowan, fixes & updates May 2001. + Sept 2001: fixed const & error conditions per + mods suggested by S. Parent & A. Lillich. + June 2002: Tim Dodd added detection and handling of incomplete + source sequences, enhanced error detection, added casts + to eliminate compiler warnings. + July 2003: slight mods to back out aggressive FFFE detection. + Jan 2004: updated switches in from-UTF8 conversions. + Oct 2004: updated to use UNI_MAX_LEGAL_UTF32 in UTF-32 conversions. + + See the header file "ConvertUTF.h" for complete documentation. + +------------------------------------------------------------------------ */ + + +#include "ConvertUTF.h" +#ifdef CVTUTF_DEBUG +#include +#endif + +static const int halfShift = 10; /* used for shifting by 10 bits */ + +static const UTF32 halfBase = 0x0010000UL; +static const UTF32 halfMask = 0x3FFUL; + +#define UNI_SUR_HIGH_START (UTF32)0xD800 +#define UNI_SUR_HIGH_END (UTF32)0xDBFF +#define UNI_SUR_LOW_START (UTF32)0xDC00 +#define UNI_SUR_LOW_END (UTF32)0xDFFF +#define false 0 +#define true 1 + +/* --------------------------------------------------------------------- */ + +ConversionResult ConvertUTF32toUTF16 ( + const UTF32** sourceStart, const UTF32* sourceEnd, + UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) { + ConversionResult result = conversionOK; + const UTF32* source = *sourceStart; + UTF16* target = *targetStart; + while (source < sourceEnd) { + UTF32 ch; + if (target >= targetEnd) { + result = targetExhausted; break; + } + ch = *source++; + if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */ + /* UTF-16 surrogate values are illegal in UTF-32; 0xffff or 0xfffe are both reserved values */ + if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { + if (flags == strictConversion) { + --source; /* return to the illegal value itself */ + result = sourceIllegal; + break; + } else { + *target++ = UNI_REPLACEMENT_CHAR; + } + } else { + *target++ = (UTF16)ch; /* normal case */ + } + } else if (ch > UNI_MAX_LEGAL_UTF32) { + if (flags == strictConversion) { + result = sourceIllegal; + } else { + *target++ = UNI_REPLACEMENT_CHAR; + } + } else { + /* target is a character in range 0xFFFF - 0x10FFFF. */ + if (target + 1 >= targetEnd) { + --source; /* Back up source pointer! */ + result = targetExhausted; break; + } + ch -= halfBase; + *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START); + *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START); + } + } + *sourceStart = source; + *targetStart = target; + return result; +} + +/* --------------------------------------------------------------------- */ + +ConversionResult ConvertUTF16toUTF32 ( + const UTF16** sourceStart, const UTF16* sourceEnd, + UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) { + ConversionResult result = conversionOK; + const UTF16* source = *sourceStart; + UTF32* target = *targetStart; + UTF32 ch, ch2; + while (source < sourceEnd) { + const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */ + ch = *source++; + /* If we have a surrogate pair, convert to UTF32 first. */ + if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) { + /* If the 16 bits following the high surrogate are in the source buffer... */ + if (source < sourceEnd) { + ch2 = *source; + /* If it's a low surrogate, convert to UTF32. */ + if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) { + ch = ((ch - UNI_SUR_HIGH_START) << halfShift) + + (ch2 - UNI_SUR_LOW_START) + halfBase; + ++source; + } else if (flags == strictConversion) { /* it's an unpaired high surrogate */ + --source; /* return to the illegal value itself */ + result = sourceIllegal; + break; + } + } else { /* We don't have the 16 bits following the high surrogate. */ + --source; /* return to the high surrogate */ + result = sourceExhausted; + break; + } + } else if (flags == strictConversion) { + /* UTF-16 surrogate values are illegal in UTF-32 */ + if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) { + --source; /* return to the illegal value itself */ + result = sourceIllegal; + break; + } + } + if (target >= targetEnd) { + source = oldSource; /* Back up source pointer! */ + result = targetExhausted; break; + } + *target++ = ch; + } + *sourceStart = source; + *targetStart = target; +#ifdef CVTUTF_DEBUG +if (result == sourceIllegal) { + fprintf(stderr, "ConvertUTF16toUTF32 illegal seq 0x%04x,%04x\n", ch, ch2); + fflush(stderr); +} +#endif + return result; +} + +/* --------------------------------------------------------------------- */ + +/* + * Index into the table below with the first byte of a UTF-8 sequence to + * get the number of trailing bytes that are supposed to follow it. + * Note that *legal* UTF-8 values can't have 4 or 5-bytes. The table is + * left as-is for anyone who may want to do such conversion, which was + * allowed in earlier algorithms. + */ +static const char trailingBytesForUTF8[256] = { + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, + 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, + 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 3,3,3,3,3,3,3,3,4,4,4,4,5,5,5,5 +}; + +/* + * Magic values subtracted from a buffer value during UTF8 conversion. + * This table contains as many values as there might be trailing bytes + * in a UTF-8 sequence. + */ +static const UTF32 offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL, + 0x03C82080UL, 0xFA082080UL, 0x82082080UL }; + +/* + * Once the bits are split out into bytes of UTF-8, this is a mask OR-ed + * into the first byte, depending on how many bytes follow. There are + * as many entries in this table as there are UTF-8 sequence types. + * (I.e., one byte sequence, two byte... etc.). Remember that sequencs + * for *legal* UTF-8 will be 4 or fewer bytes total. + */ +static const UTF8 firstByteMark[7] = { 0x00, 0x00, 0xC0, 0xE0, 0xF0, 0xF8, 0xFC }; + +/* --------------------------------------------------------------------- */ + +/* The interface converts a whole buffer to avoid function-call overhead. + * Constants have been gathered. Loops & conditionals have been removed as + * much as possible for efficiency, in favor of drop-through switches. + * (See "Note A" at the bottom of the file for equivalent code.) + * If your compiler supports it, the "isLegalUTF8" call can be turned + * into an inline function. + */ + +/* --------------------------------------------------------------------- */ + +ConversionResult ConvertUTF16toUTF8 ( + const UTF16** sourceStart, const UTF16* sourceEnd, + UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) { + ConversionResult result = conversionOK; + const UTF16* source = *sourceStart; + UTF8* target = *targetStart; + while (source < sourceEnd) { + UTF32 ch; + unsigned short bytesToWrite = 0; + const UTF32 byteMask = 0xBF; + const UTF32 byteMark = 0x80; + const UTF16* oldSource = source; /* In case we have to back up because of target overflow. */ + ch = *source++; + /* If we have a surrogate pair, convert to UTF32 first. */ + if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_HIGH_END) { + /* If the 16 bits following the high surrogate are in the source buffer... */ + if (source < sourceEnd) { + UTF32 ch2 = *source; + /* If it's a low surrogate, convert to UTF32. */ + if (ch2 >= UNI_SUR_LOW_START && ch2 <= UNI_SUR_LOW_END) { + ch = ((ch - UNI_SUR_HIGH_START) << halfShift) + + (ch2 - UNI_SUR_LOW_START) + halfBase; + ++source; + } else if (flags == strictConversion) { /* it's an unpaired high surrogate */ + --source; /* return to the illegal value itself */ + result = sourceIllegal; + break; + } + } else { /* We don't have the 16 bits following the high surrogate. */ + --source; /* return to the high surrogate */ + result = sourceExhausted; + break; + } + } else if (flags == strictConversion) { + /* UTF-16 surrogate values are illegal in UTF-32 */ + if (ch >= UNI_SUR_LOW_START && ch <= UNI_SUR_LOW_END) { + --source; /* return to the illegal value itself */ + result = sourceIllegal; + break; + } + } + /* Figure out how many bytes the result will require */ + if (ch < (UTF32)0x80) { bytesToWrite = 1; + } else if (ch < (UTF32)0x800) { bytesToWrite = 2; + } else if (ch < (UTF32)0x10000) { bytesToWrite = 3; + } else if (ch < (UTF32)0x110000) { bytesToWrite = 4; + } else { bytesToWrite = 3; + ch = UNI_REPLACEMENT_CHAR; + } + + target += bytesToWrite; + if (target > targetEnd) { + source = oldSource; /* Back up source pointer! */ + target -= bytesToWrite; result = targetExhausted; break; + } + switch (bytesToWrite) { /* note: everything falls through. */ + case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; + case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; + case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; + case 1: *--target = (UTF8)(ch | firstByteMark[bytesToWrite]); + } + target += bytesToWrite; + } + *sourceStart = source; + *targetStart = target; + return result; +} + +/* --------------------------------------------------------------------- */ + +/* + * Utility routine to tell whether a sequence of bytes is legal UTF-8. + * This must be called with the length pre-determined by the first byte. + * If not calling this from ConvertUTF8to*, then the length can be set by: + * length = trailingBytesForUTF8[*source]+1; + * and the sequence is illegal right away if there aren't that many bytes + * available. + * If presented with a length > 4, this returns false. The Unicode + * definition of UTF-8 goes up to 4-byte sequences. + */ + +static Boolean isLegalUTF8(const UTF8 *source, int length) { + UTF8 a; + const UTF8 *srcptr = source+length; + switch (length) { + default: return false; + /* Everything else falls through when "true"... */ + case 4: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; + case 3: if ((a = (*--srcptr)) < 0x80 || a > 0xBF) return false; + case 2: if ((a = (*--srcptr)) > 0xBF) return false; + + switch (*source) { + /* no fall-through in this inner switch */ + case 0xE0: if (a < 0xA0) return false; break; + case 0xED: if (a > 0x9F) return false; break; + case 0xF0: if (a < 0x90) return false; break; + case 0xF4: if (a > 0x8F) return false; break; + default: if (a < 0x80) return false; + } + + case 1: if (*source >= 0x80 && *source < 0xC2) return false; + } + if (*source > 0xF4) return false; + return true; +} + +/* --------------------------------------------------------------------- */ + +/* + * Exported function to return whether a UTF-8 sequence is legal or not. + * This is not used here; it's just exported. + */ +Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd) { + int length = trailingBytesForUTF8[*source]+1; + if (source+length > sourceEnd) { + return false; + } + return isLegalUTF8(source, length); +} + +/* --------------------------------------------------------------------- */ + +ConversionResult ConvertUTF8toUTF16 ( + const UTF8** sourceStart, const UTF8* sourceEnd, + UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags) { + ConversionResult result = conversionOK; + const UTF8* source = *sourceStart; + UTF16* target = *targetStart; + while (source < sourceEnd) { + UTF32 ch = 0; + unsigned short extraBytesToRead = trailingBytesForUTF8[*source]; + if (source + extraBytesToRead >= sourceEnd) { + result = sourceExhausted; break; + } + /* Do this check whether lenient or strict */ + if (! isLegalUTF8(source, extraBytesToRead+1)) { + result = sourceIllegal; + break; + } + /* + * The cases all fall through. See "Note A" below. + */ + switch (extraBytesToRead) { + case 5: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */ + case 4: ch += *source++; ch <<= 6; /* remember, illegal UTF-8 */ + case 3: ch += *source++; ch <<= 6; + case 2: ch += *source++; ch <<= 6; + case 1: ch += *source++; ch <<= 6; + case 0: ch += *source++; + } + ch -= offsetsFromUTF8[extraBytesToRead]; + + if (target >= targetEnd) { + source -= (extraBytesToRead+1); /* Back up source pointer! */ + result = targetExhausted; break; + } + if (ch <= UNI_MAX_BMP) { /* Target is a character <= 0xFFFF */ + /* UTF-16 surrogate values are illegal in UTF-32 */ + if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { + if (flags == strictConversion) { + source -= (extraBytesToRead+1); /* return to the illegal value itself */ + result = sourceIllegal; + break; + } else { + *target++ = UNI_REPLACEMENT_CHAR; + } + } else { + *target++ = (UTF16)ch; /* normal case */ + } + } else if (ch > UNI_MAX_UTF16) { + if (flags == strictConversion) { + result = sourceIllegal; + source -= (extraBytesToRead+1); /* return to the start */ + break; /* Bail out; shouldn't continue */ + } else { + *target++ = UNI_REPLACEMENT_CHAR; + } + } else { + /* target is a character in range 0xFFFF - 0x10FFFF. */ + if (target + 1 >= targetEnd) { + source -= (extraBytesToRead+1); /* Back up source pointer! */ + result = targetExhausted; break; + } + ch -= halfBase; + *target++ = (UTF16)((ch >> halfShift) + UNI_SUR_HIGH_START); + *target++ = (UTF16)((ch & halfMask) + UNI_SUR_LOW_START); + } + } + *sourceStart = source; + *targetStart = target; + return result; +} + +/* --------------------------------------------------------------------- */ + +ConversionResult ConvertUTF32toUTF8 ( + const UTF32** sourceStart, const UTF32* sourceEnd, + UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags) { + ConversionResult result = conversionOK; + const UTF32* source = *sourceStart; + UTF8* target = *targetStart; + while (source < sourceEnd) { + UTF32 ch; + unsigned short bytesToWrite = 0; + const UTF32 byteMask = 0xBF; + const UTF32 byteMark = 0x80; + ch = *source++; + if (flags == strictConversion ) { + /* UTF-16 surrogate values are illegal in UTF-32 */ + if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { + --source; /* return to the illegal value itself */ + result = sourceIllegal; + break; + } + } + /* + * Figure out how many bytes the result will require. Turn any + * illegally large UTF32 things (> Plane 17) into replacement chars. + */ + if (ch < (UTF32)0x80) { bytesToWrite = 1; + } else if (ch < (UTF32)0x800) { bytesToWrite = 2; + } else if (ch < (UTF32)0x10000) { bytesToWrite = 3; + } else if (ch <= UNI_MAX_LEGAL_UTF32) { bytesToWrite = 4; + } else { bytesToWrite = 3; + ch = UNI_REPLACEMENT_CHAR; + result = sourceIllegal; + } + + target += bytesToWrite; + if (target > targetEnd) { + --source; /* Back up source pointer! */ + target -= bytesToWrite; result = targetExhausted; break; + } + switch (bytesToWrite) { /* note: everything falls through. */ + case 4: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; + case 3: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; + case 2: *--target = (UTF8)((ch | byteMark) & byteMask); ch >>= 6; + case 1: *--target = (UTF8) (ch | firstByteMark[bytesToWrite]); + } + target += bytesToWrite; + } + *sourceStart = source; + *targetStart = target; + return result; +} + +/* --------------------------------------------------------------------- */ + +ConversionResult ConvertUTF8toUTF32 ( + const UTF8** sourceStart, const UTF8* sourceEnd, + UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags) { + ConversionResult result = conversionOK; + const UTF8* source = *sourceStart; + UTF32* target = *targetStart; + while (source < sourceEnd) { + UTF32 ch = 0; + unsigned short extraBytesToRead = trailingBytesForUTF8[*source]; + if (source + extraBytesToRead >= sourceEnd) { + result = sourceExhausted; break; + } + /* Do this check whether lenient or strict */ + if (! isLegalUTF8(source, extraBytesToRead+1)) { + result = sourceIllegal; + break; + } + /* + * The cases all fall through. See "Note A" below. + */ + switch (extraBytesToRead) { + case 5: ch += *source++; ch <<= 6; + case 4: ch += *source++; ch <<= 6; + case 3: ch += *source++; ch <<= 6; + case 2: ch += *source++; ch <<= 6; + case 1: ch += *source++; ch <<= 6; + case 0: ch += *source++; + } + ch -= offsetsFromUTF8[extraBytesToRead]; + + if (target >= targetEnd) { + source -= (extraBytesToRead+1); /* Back up the source pointer! */ + result = targetExhausted; break; + } + if (ch <= UNI_MAX_LEGAL_UTF32) { + /* + * UTF-16 surrogate values are illegal in UTF-32, and anything + * over Plane 17 (> 0x10FFFF) is illegal. + */ + if (ch >= UNI_SUR_HIGH_START && ch <= UNI_SUR_LOW_END) { + if (flags == strictConversion) { + source -= (extraBytesToRead+1); /* return to the illegal value itself */ + result = sourceIllegal; + break; + } else { + *target++ = UNI_REPLACEMENT_CHAR; + } + } else { + *target++ = ch; + } + } else { /* i.e., ch > UNI_MAX_LEGAL_UTF32 */ + result = sourceIllegal; + *target++ = UNI_REPLACEMENT_CHAR; + } + } + *sourceStart = source; + *targetStart = target; + return result; +} + +/* --------------------------------------------------------------------- + + Note A. + The fall-through switches in UTF-8 reading code save a + temp variable, some decrements & conditionals. The switches + are equivalent to the following loop: + { + int tmpBytesToRead = extraBytesToRead+1; + do { + ch += *source++; + --tmpBytesToRead; + if (tmpBytesToRead) ch <<= 6; + } while (tmpBytesToRead > 0); + } + In UTF-8 writing code, the switches on "bytesToWrite" are + similarly unrolled loops. + + --------------------------------------------------------------------- */ diff -r 000000000000 -r 582cbe817f2c src/Support/ConvertUTF.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/Support/ConvertUTF.h Fri Feb 12 12:31:23 2010 +0000 @@ -0,0 +1,149 @@ +/* + * Copyright 2001-2004 Unicode, Inc. + * + * Disclaimer + * + * This source code is provided as is by Unicode, Inc. No claims are + * made as to fitness for any particular purpose. No warranties of any + * kind are expressed or implied. The recipient agrees to determine + * applicability of information provided. If this file has been + * purchased on magnetic or optical media from Unicode, Inc., the + * sole remedy for any claim will be exchange of defective media + * within 90 days of receipt. + * + * Limitations on Rights to Redistribute This Code + * + * Unicode, Inc. hereby grants the right to freely use the information + * supplied in this file in the creation of products supporting the + * Unicode Standard, and to make copies of this file in any form + * for internal or external distribution as long as this notice + * remains attached. + */ + +/* --------------------------------------------------------------------- + + Conversions between UTF32, UTF-16, and UTF-8. Header file. + + Several funtions are included here, forming a complete set of + conversions between the three formats. UTF-7 is not included + here, but is handled in a separate source file. + + Each of these routines takes pointers to input buffers and output + buffers. The input buffers are const. + + Each routine converts the text between *sourceStart and sourceEnd, + putting the result into the buffer between *targetStart and + targetEnd. Note: the end pointers are *after* the last item: e.g. + *(sourceEnd - 1) is the last item. + + The return result indicates whether the conversion was successful, + and if not, whether the problem was in the source or target buffers. + (Only the first encountered problem is indicated.) + + After the conversion, *sourceStart and *targetStart are both + updated to point to the end of last text successfully converted in + the respective buffers. + + Input parameters: + sourceStart - pointer to a pointer to the source buffer. + The contents of this are modified on return so that + it points at the next thing to be converted. + targetStart - similarly, pointer to pointer to the target buffer. + sourceEnd, targetEnd - respectively pointers to the ends of the + two buffers, for overflow checking only. + + These conversion functions take a ConversionFlags argument. When this + flag is set to strict, both irregular sequences and isolated surrogates + will cause an error. When the flag is set to lenient, both irregular + sequences and isolated surrogates are converted. + + Whether the flag is strict or lenient, all illegal sequences will cause + an error return. This includes sequences such as: , , + or in UTF-8, and values above 0x10FFFF in UTF-32. Conformant code + must check for illegal sequences. + + When the flag is set to lenient, characters over 0x10FFFF are converted + to the replacement character; otherwise (when the flag is set to strict) + they constitute an error. + + Output parameters: + The value "sourceIllegal" is returned from some routines if the input + sequence is malformed. When "sourceIllegal" is returned, the source + value will point to the illegal value that caused the problem. E.g., + in UTF-8 when a sequence is malformed, it points to the start of the + malformed sequence. + + Author: Mark E. Davis, 1994. + Rev History: Rick McGowan, fixes & updates May 2001. + Fixes & updates, Sept 2001. + +------------------------------------------------------------------------ */ + +/* --------------------------------------------------------------------- + The following 4 definitions are compiler-specific. + The C standard does not guarantee that wchar_t has at least + 16 bits, so wchar_t is no less portable than unsigned short! + All should be unsigned values to avoid sign extension during + bit mask & shift operations. +------------------------------------------------------------------------ */ + +typedef unsigned int UTF32; /* at least 32 bits */ +typedef unsigned short UTF16; /* at least 16 bits */ +typedef unsigned char UTF8; /* typically 8 bits */ +typedef unsigned char Boolean; /* 0 or 1 */ + +/* Some fundamental constants */ +#define UNI_REPLACEMENT_CHAR (UTF32)0x0000FFFD +#define UNI_MAX_BMP (UTF32)0x0000FFFF +#define UNI_MAX_UTF16 (UTF32)0x0010FFFF +#define UNI_MAX_UTF32 (UTF32)0x7FFFFFFF +#define UNI_MAX_LEGAL_UTF32 (UTF32)0x0010FFFF + +typedef enum { + conversionOK, /* conversion successful */ + sourceExhausted, /* partial character in source, but hit end */ + targetExhausted, /* insuff. room in target for conversion */ + sourceIllegal /* source sequence is illegal/malformed */ +} ConversionResult; + +typedef enum { + strictConversion = 0, + lenientConversion +} ConversionFlags; + +/* This is for C++ and does no harm in C */ +#ifdef __cplusplus +extern "C" { +#endif + +ConversionResult ConvertUTF8toUTF16 ( + const UTF8** sourceStart, const UTF8* sourceEnd, + UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags); + +ConversionResult ConvertUTF16toUTF8 ( + const UTF16** sourceStart, const UTF16* sourceEnd, + UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags); + +ConversionResult ConvertUTF8toUTF32 ( + const UTF8** sourceStart, const UTF8* sourceEnd, + UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags); + +ConversionResult ConvertUTF32toUTF8 ( + const UTF32** sourceStart, const UTF32* sourceEnd, + UTF8** targetStart, UTF8* targetEnd, ConversionFlags flags); + +ConversionResult ConvertUTF16toUTF32 ( + const UTF16** sourceStart, const UTF16* sourceEnd, + UTF32** targetStart, UTF32* targetEnd, ConversionFlags flags); + +ConversionResult ConvertUTF32toUTF16 ( + const UTF32** sourceStart, const UTF32* sourceEnd, + UTF16** targetStart, UTF16* targetEnd, ConversionFlags flags); + +Boolean isLegalUTF8Sequence(const UTF8 *source, const UTF8 *sourceEnd); + +#ifdef __cplusplus +} +#endif + +/* --------------------------------------------------------------------- */ diff -r 000000000000 -r 582cbe817f2c src/Support/ERBTools.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/Support/ERBTools.h Fri Feb 12 12:31:23 2010 +0000 @@ -0,0 +1,46 @@ +// Copyright 2006-2010, Thomas Walters +// +// AIM-C: A C++ implementation of the Auditory Image Model +// http://www.acousticscale.org/AIMC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +/*! \file + * \brief + */ + +/*! \author: Thomas Walters + * \date 2010/01/23 + * \version \$Id: ERBTools.h 1 2010-02-02 11:04:50Z tcw $ + */ + +#ifndef _AIMC_SUPPORT_ERBTOOLS_H_ +#define _AIMC_SUPPORT_ERBTOOLS_H_ + +#include + +namespace aimc { +class ERBTools { + public: + static float Freq2ERB(float freq) { + return 21.4f * log10(4.37f * freq / 1000.0f + 1.0f); + } + + static float Freq2ERBw(float freq) { + return 24.7f * (4.37f * freq / 1000.0f + 1.0f); + } +}; +} + +#endif // _AIMC_SUPPORT_ERBTOOLS_H_ diff -r 000000000000 -r 582cbe817f2c src/Support/Module.cc --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/Support/Module.cc Fri Feb 12 12:31:23 2010 +0000 @@ -0,0 +1,117 @@ +// Copyright 2010, Thomas Walters +// +// AIM-C: A C++ implementation of the Auditory Image Model +// http://www.acousticscale.org/AIMC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +/*! \file + * \brief Base class for all AIM-C modules. + */ + +/*! \author: Thomas Walters + * \date 2010/01/23 + * \version \$Id$ + */ + +#include "Support/Module.h" + +#include + +namespace aimc { +using std::pair; +Module::Module(Parameters *parameters) { + initialized_ = false; + targets_.clear(); + parameters_ = parameters; + module_identifier_ = "MODULE IDENTIFIER NOT SET"; + module_type_ = "MODULE TYPE NOT SET"; + module_description_ = "MODULE DESCRIPTION NOT SET"; + module_version_ = "MODULE VERSION NOT SET"; +}; + +Module::~Module() { +}; + +bool Module::Initialize(const SignalBank &input) { + // Validate the input + if (!input.Validate()) { + LOG_ERROR("Input SignalBank not valid"); + return false; + } + if (!InitializeInternal(input)) { + LOG_ERROR("Module initialization failed"); + return false; + } + // If the module has an output bank, then we can set up the targets + // of this module.. + if (output_.initialized()) { + // Check that the output SignalBank has been set up correctly + if (!output_.Validate()) { + LOG_ERROR("Output SignalBank not valid"); + return false; + } + // Iterate through all the targets of this module, initializing + // them if they have not already been initialized. If they have + // already been initialized, then they are assumed to have been + // set up to accept an input SignalBank of the correct form, but + // this is not checked. + set::const_iterator it; + for (it = targets_.begin(); it != targets_.end(); ++it) { + if (!(*it)->initialized()) + if (!(*it)->Initialize(output_)) + return false; + } + } + initialized_ = true; + return true; +} + +bool Module::initialized() const { + return initialized_; +} + +bool Module::AddTarget(Module* target_module) { + if (target_module) { + pair::iterator, bool> ret; + ret = targets_.insert(target_module); + return ret.second; + } + return false; +} + +bool Module::DeleteTarget(Module* target_module) { + if (targets_.erase(target_module) != 0) + return true; + return false; +} + +void Module::DeleteAllTargets() { + targets_.clear(); +} + +const SignalBank* Module::GetOutputBank() const { + return &output_; +} + +void Module::PushOutput() { + if (output_.initialized()) { + set::const_iterator it; + for (it = targets_.begin(); it != targets_.end(); ++it) { + (*it)->Process(output_); + } + } +} +} // namespace aimc + diff -r 000000000000 -r 582cbe817f2c src/Support/Module.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/Support/Module.h Fri Feb 12 12:31:23 2010 +0000 @@ -0,0 +1,130 @@ +// Copyright 2010, Thomas Walters +// +// AIM-C: A C++ implementation of the Auditory Image Model +// http://www.acousticscale.org/AIMC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +/*! \file + * \brief Base class for all AIM-C modules. + */ + +/*! The module construcor is called with a pointer to a set of Parameters. + * In the constructor, the module sets the defaults for its various + * parameters. + * A module is initialized with a pointer to a valid SignalBank + * (source modules can be initialized with the NULL pointer). After the + * Initialize(SignalBank*) function has been called, a call to GetOutputBank() + * returns a pointer to a SignalBank in which the results + * of the module's processing will be placed. Modules can use the output_ + * SignalBank to store their output, or leave it uninitialized if they do not + * produce an output. + * At each call to Process(input), the module takes the + * SignalBank 'input' (which must, unless otherwise specified, have the same + * number of channels, sample rate, buffer size and centre frequencies as the + * SignalBank which was passed to Initialize()), processes it, and places the + * output in the internal SignalBank output_. + * Modules can have an arbitrary number of unique targets. Each + * completed output frame is 'pushed' to all of the targets of the module + * in turn when PushOutput() is called. To achieve this, after each complete + * output SignalBank is filled, the module calls the Process() function of + * each of its targets in turn. + * When Initialize() is first called. The module Initialize()s all of its + * targets with its ouptut_ SignalBank, if it's output bank has been set up. + */ + +/*! \author: Thomas Walters + * \date 2010/01/23 + * \version \$Id: Module.h 4 2010-02-03 18:44:58Z tcw $ + */ + +#ifndef _AIMC_SUPPORT_MODULE_H_ +#define _AIMC_SUPPORT_MODULE_H_ + +#include +#include + +#include "Support/Common.h" +#include "Support/Parameters.h" +#include "Support/SignalBank.h" + +namespace aimc { +using std::set; +using std::string; +class Module { + public: + explicit Module(Parameters *parameters); + + virtual ~Module(); + + /* \brief Validate this module's output SignalBank, and initialize + * any targets of the module if necessary. + * \param input Input SignalBank. + * \param output true on success, false on failure. + */ + bool Initialize(const SignalBank &input); + + /*! \brief + */ + bool initialized() const; + + /* \brief Add a target to this module. Whenever it generates a new + * output, this module will push its output to all its targets. + * \param input Target module to add. + * \param output true on success, false on failure. + */ + bool AddTarget(Module* target_module); + + /*! \brief + */ + bool DeleteTarget(Module* target_module); + + /*! \brief + */ + void DeleteAllTargets(); + + /*! \brief Process a buffer + */ + virtual void Process(const SignalBank &input) = 0; + + /*! \brief Reset the internal state of the module to that when it was + * initialised + */ + virtual void Reset() = 0; + + /*! \brief + */ + const SignalBank* GetOutputBank() const; + + protected: + void PushOutput(); + + virtual bool InitializeInternal(const SignalBank &input) = 0; + + bool initialized_; + set targets_; + SignalBank output_; + Parameters* parameters_; + + string module_identifier_; + string module_type_; + string module_description_; + string module_version_; + + private: + DISALLOW_COPY_AND_ASSIGN(Module); +}; +} + +#endif // _AIMC_SUPPORT_MODULE_H_ diff -r 000000000000 -r 582cbe817f2c src/Support/Parameters.cc --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/Support/Parameters.cc Fri Feb 12 12:31:23 2010 +0000 @@ -0,0 +1,267 @@ +// Copyright 2006-2010, Willem van Engen +// +// AIM-C: A C++ implementation of the Auditory Image Model +// http://www.acousticscale.org/AIMC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! +//! \file +//! \brief Main parameters store +//! +//! \author Willem van Engen +//! \date created 2006/09/21 +//! \version \$Id: Parameters.cc 4 2010-02-03 18:44:58Z tcw $ + +#include +#include +#include + +#include "Support/Common.h" +#include "Support/Parameters.h" + +namespace aimc { +const char *Parameters::m_SDefaultIniSection = ""; + +Parameters::Parameters() { + m_iNestCount = 0; + m_pIni = new CSimpleIniCase(false, false, true); + AIM_ASSERT(m_pIni); +} + +Parameters::~Parameters() { + DELETE_IF_NONNULL(m_pIni); +} + +const char * Parameters::DefaultString(const char* sName, const char* val) { + AIM_ASSERT(m_pIni); + if (!IsSet(sName)) { + m_pIni->SetValue(m_SDefaultIniSection, sName, val); + } + return m_pIni->GetString(sName); +} + +int Parameters::DefaultInt(const char* sName, int val) { + AIM_ASSERT(m_pIni); + if (!IsSet(sName)) { + m_pIni->SetInt(m_SDefaultIniSection, sName, val); + } + return m_pIni->GetInt(sName); +} + +unsigned int Parameters::DefaultUInt(const char* sName, unsigned int val) { + AIM_ASSERT(m_pIni); + if (!IsSet(sName)) { + m_pIni->SetUInt(m_SDefaultIniSection, sName, val); + } + return m_pIni->GetUInt(sName); +} + +float Parameters::DefaultFloat(const char* sName, float val) { + AIM_ASSERT(m_pIni); + if (!IsSet(sName)) { + m_pIni->SetFloat(m_SDefaultIniSection, sName, val); + } + return m_pIni->GetFloat(sName); +} + +bool Parameters::DefaultBool(const char* sName, bool val) { + AIM_ASSERT(m_pIni); + if (!IsSet(sName)) { + m_pIni->SetBool(m_SDefaultIniSection, sName, val); + } + return m_pIni->GetBool(sName); +} + +void Parameters::SetString(const char *sName, const char *val) { + AIM_ASSERT(m_pIni); + m_pIni->SetValue(m_SDefaultIniSection, sName, val); +} + +void Parameters::SetInt(const char *sName, int val) { + char sVal[20]; + snprintf(sVal, sizeof(sVal)/sizeof(sVal[0]), "%d", val); + SetString(sName, sVal); +} + +void Parameters::SetUInt(const char *sName, unsigned int val) { + char sVal[20]; + snprintf(sVal, sizeof(sVal)/sizeof(sVal[0]), "%ud", val); + SetString(sName, sVal); +} + +void Parameters::SetBool(const char *sName, bool val) { + SetString(sName, val ? "true" : "false"); +} + +void Parameters::SetFloat(const char *sName, float val) { + char sVal[20]; + snprintf(sVal, sizeof(sVal)/sizeof(sVal[0]), "%f", val); + SetString(sName, sVal); +} + +const char *Parameters::GetString(const char *sName) { + AIM_ASSERT(m_pIni); + const char *sVal = m_pIni->GetValue(m_SDefaultIniSection, sName, NULL); + if (!sVal) { + LOG_ERROR(_T("Parameter not found '%s'"), sName); + return ""; + } + return sVal; +} + +int Parameters::GetInt(const char *sName) { + return atoi(GetString(sName)); +} + +unsigned int Parameters::GetUInt(const char *sName) { + return atoi(GetString(sName)); +} + +float Parameters::GetFloat(const char *sName) { + return (float)atof(GetString(sName)); +} + +bool Parameters::GetBool(const char *sName) { + const char *sVal = GetString(sName); + if (strcmp(sVal, "true")==0 || strcmp(sVal, "on")==0 || + strcmp(sVal, "yes")==0 || strcmp(sVal, "1")==0 || + strcmp(sVal, "y")==0 || strcmp(sVal, "t")==0) + return true; + else + return false; +} + +bool Parameters::IsSet(const char *sName) { + AIM_ASSERT(m_pIni); + return m_pIni->GetValue(m_SDefaultIniSection, sName, NULL)!=NULL; +} + +bool Parameters::Parse(const char *sCmd) { + //! \todo There is some code duplication here from Parameters::Merge() + + CSimpleIniCase *pIni2 = new CSimpleIniCase(false, false, true); + AIM_ASSERT(pIni2); + if (pIni2->Load(sCmd, strlen(sCmd)) < 0) { + LOG_ERROR(_T("Could not parse option '%s'"), sCmd); + delete pIni2; + return false; + } + + // if there are keys and values... + const CSimpleIniCase::TKeyVal *pSectionData = + pIni2->GetSection(m_SDefaultIniSection); + if (pSectionData) { + // iterate over all keys and set them in the current parameter file + CSimpleIniCase::TKeyVal::const_iterator iKeyVal = pSectionData->begin(); + for ( ;iKeyVal != pSectionData->end(); ++iKeyVal) { + m_pIni->SetValue(m_SDefaultIniSection, + iKeyVal->first.pItem, + iKeyVal->second); + } + } + delete pIni2; + return true; +} + +bool Parameters::Delete(const char *sName) { + AIM_ASSERT(m_pIni); + return(m_pIni->Delete(m_SDefaultIniSection, sName)); +} + +bool Parameters::LoadFile(const char *sParamFilename) { + AIM_ASSERT(m_pIni); + SI_Error siErr; + bool bRet = true; + + // Avoid inclusion loops + if (m_iNestCount >= m_iNestCountMaximum) { + LOG_ERROR(_T("Possible inclusion loop in file '%s' (%d times)"), + sParamFilename, m_iNestCount); + return false; + } + m_iNestCount++; + + if ( (siErr=m_pIni->LoadFile(sParamFilename))<0 ) { + // Don't complain if file not found, but do return error + if (siErr!=SI_FILE) + LOG_ERROR(_T("Couldn't parse parameters from '%s'"), sParamFilename); + m_iNestCount--; + return false; + } + + m_iNestCount--; + return bRet; +} + +bool Parameters::MergeFile(const char *sParamFilename) { + Parameters *pParam2 = new Parameters(); + AIM_ASSERT(pParam2); + if (!pParam2->LoadFile(sParamFilename)) { + LOG_ERROR(_T("Could not load parameter file '%s' for merging"), + sParamFilename); + delete pParam2; + return false; + } + + // if there are keys and values... + const CSimpleIniCase::TKeyVal *pSectionData = + pParam2->m_pIni->GetSection(m_SDefaultIniSection); + if (pSectionData) { + // iterate over all keys and set them in the current parameter file + CSimpleIniCase::TKeyVal::const_iterator iKeyVal = pSectionData->begin(); + for ( ;iKeyVal != pSectionData->end(); ++iKeyVal) { + m_pIni->SetValue(m_SDefaultIniSection, + iKeyVal->first.pItem, + iKeyVal->second); + } + } + + delete pParam2; + return true; +} + +bool Parameters::Merge(const char *sParamFilename) { + return MergeFile(sParamFilename); +} + +bool Parameters::Load(const char *sParamFilename) { + bool bRet = true; + // Load parameter file + bRet &= Merge(sParamFilename); + return bRet; +} + +bool Parameters::Save(const char *sParamFilename) { + AIM_ASSERT(m_pIni); + SI_Error siErr; + FILE *pSaveFile; + + pSaveFile = fopen(sParamFilename, "wb"); + if (pSaveFile == NULL) { + LOG_ERROR(_T("Couldn't create parameters file '%s' to save to"), + sParamFilename); + return false; + } + + if ((siErr = m_pIni->SaveFile(pSaveFile)) < 0 ) { + LOG_ERROR(_T("Couldn't save parameters in file '%s'"), sParamFilename); + return false; + } + + fclose(pSaveFile); + + return true; +} +} // namespace aimc diff -r 000000000000 -r 582cbe817f2c src/Support/Parameters.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/Support/Parameters.h Fri Feb 12 12:31:23 2010 +0000 @@ -0,0 +1,208 @@ +// Copyright 2006-2010, Willem van Engen +// +// AIM-C: A C++ implementation of the Auditory Image Model +// http://www.acousticscale.org/AIMC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +//! +//! \file +//! \brief Main parameters store +//! +//! \author Willem van Engen +//! \date created 2006/09/21 +//! \version \$Id: Parameters.h 4 2010-02-03 18:44:58Z tcw $ + +#ifndef _AIMC_SUPPORT_PARAMETERS_H_ +#define _AIMC_SUPPORT_PARAMETERS_H_ + +// If not _WINDOWS, please compile in Support/ConvertUTF.c +#ifdef _UNICODE +// Here we want to use the ANSI version of all the non wxWidgets stuff, but +// convert stribngs to Unicode when used in wxWidgets. This allows all the +// string handling in the non-GUI version to use ANSI text only, but to pass +// unicode text to the GUI +#undef _UNICODE +#include "Support/SimpleIni.h" +#define _UNICODE +#else +#include "Support/SimpleIni.h" +#endif + +namespace aimc { +/*! + * \class Parameters "Support/Parameters.h" + * \brief Main parameter store for parameters + */ +class Parameters { +public: + Parameters(); + ~Parameters(); + + /*! + * \brief Load parameter file + * \param sParamFilename Filename of parameter file to read + * \return true on succes, false on error + * + */ + bool Load(const char *sParamFilename); + + /*! \brief Save Parameter File + * \param sParamFilename Filename of parameter file to save + * \return true on success, false on error + */ + bool Save(const char *sParamFilename); + + /*! + * \brief Load parameter file and merge parameters with current, overwriting + * duplicates. + * \param sParamFilename Filename of parameter file to read + * \return true on succes, false on error + */ + bool Merge(const char *sParamFilename); + + /*! \brief Get a parameter, setting it with a default value if it is not + * already set + * \param sName Name of parameter + * \param val Value of the parameter + */ + const char * DefaultString(const char* sName, const char *val); + + /*! \overload + */ + int DefaultInt(const char* sName, int val); + + /*! \overload + */ + unsigned int DefaultUInt(const char* sName, unsigned int val); + + /*! \overload + */ + float DefaultFloat(const char* sName, float val); + + /*! \overload + */ + bool DefaultBool(const char* sName, bool val); + + + /*! \brief Set a parameter + * \param sName Name of parameter + * \param val Value of parameter + */ + void SetString(const char *sName, const char *val); + + /*! \overload + */ + void SetInt(const char *sName, int val); + + /*! \overload + */ + void SetUInt(const char *sName, unsigned int val); + + /*! \overload + */ + void SetFloat(const char *sName, float val); + + /*! \overload + */ + void SetBool(const char *sName, bool val); + + /*! \brief Get the value of a parameter + * \param sName Name of parameter + * \return Value of parameter + * + * The specified parameter _must_ exist. So put every parameter you may + * possibly require into the default parameter file. This method returns + * the empty string when a value doesn't exist, but that is only to keep + * the application from crashing: don't count on it. + * You can use IsSet() however to check for a parameter's existence. + */ + const char *GetString(const char *sName); + + /*! \overload + */ + int GetInt(const char *sName); + + /*! \overload + */ + unsigned int GetUInt(const char *sName); + + /*! \overload + */ + float GetFloat(const char *sName); + + /*! \overload + */ + bool GetBool(const char *sName); + + /*! \brief Returns if the parameters exists or not + * \param sName Name of parameter + * \return true if exists, false if not + */ + bool IsSet(const char *sName); + + /*! \brief Sets a parameter assignment from a string + * \param sCmd String to parse + * \return true if parsing succeeded, false if an error occured. + * + * This function parses a string like "foo.bar=50" and sets the parameter + * accordingly. Use this function to parse parameter assignments as given + * by the user, for example on the command-line or in a parameter file. + */ + bool Parse(const char *sCmd); + + /*! \brief Delete a parameter. GetSection may not return correctly after + * this call, so it may not be possible to repopulate the parameter grid + * after deleting a parameter + * \param sName Parameter name + * \return true on success + */ + bool Delete(const char *sName); + + //! \brief Maximum length of a parameter name in characters + static const unsigned int MaxParamNameLength = 128; + +protected: + /*! + * \brief Load parameter file + * \param sParamFilename Filename of parameter file to read + * \return true on succes, false on error + */ + bool LoadFile(const char *sParamFilename); + /*! + * \brief Load parameter file and merge parameters with current, + * overwriting duplicates. + * \param sParamFilename Filename of parameter file to read + * \return true on succes, false on error + */ + bool MergeFile(const char *sParamFilename); + + /*! \brief Default ini-section to use + * + * Since SimpleIni is an ini-parser but we don't want the sections, + * we use the empty section. This gives use the behaviour desired. + */ + static const char *m_SDefaultIniSection; + + //! \brief Parameter file object + CSimpleIniCase *m_pIni; + //! \brief \c preset.include nesting counter to avoid loops + unsigned int m_iNestCount; + //! \brief maximum value m_iNestCount may reach + static const unsigned int m_iNestCountMaximum = 16; +}; +} + +#endif // _AIMC_SUPPORT_PARAMETERS_H_ + diff -r 000000000000 -r 582cbe817f2c src/Support/SignalBank.cc --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/Support/SignalBank.cc Fri Feb 12 12:31:23 2010 +0000 @@ -0,0 +1,168 @@ +// Copyright 2010, Thomas Walters +// +// AIM-C: A C++ implementation of the Auditory Image Model +// http://www.acousticscale.org/AIMC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +/*! \file + * \brief + */ + +/*! \author: Thomas Walters + * \date 2010/01/23 + * \version \$Id$ + */ + +#include "Support/SignalBank.h" + +namespace aimc { +using std::deque; +using std::vector; + +SignalBank::SignalBank() { + sample_rate_ = 0.0f; + start_time_ = 0; + channel_count_ = 0; + buffer_length_ = 0; + initialized_ = false; +} + +SignalBank::~SignalBank() { +} + +bool SignalBank::Initialize(int channel_count, + int signal_length, + float sample_rate) { + if (channel_count < 1) + return false; + if (signal_length < 1) + return false; + if (sample_rate < 0.0f) + return false; + + start_time_ = 0; + sample_rate_ = sample_rate; + buffer_length_ = signal_length; + channel_count_ = channel_count; + signals_.resize(channel_count_); + strobes_.resize(channel_count_); + centre_frequencies_.resize(channel_count_, 0.0f); + for (int i = 0; i < channel_count_; ++i) { + signals_[i].resize(buffer_length_, 0.0f); + } + initialized_ = true; + return true; +} + +bool SignalBank::Initialize(const SignalBank &input) { + if (input.channel_count() < 1) + return false; + if (input.buffer_length() < 1) + return false; + if (input.sample_rate() < 0.0f) + return false; + + start_time_ = input.start_time(); + sample_rate_ = input.sample_rate(); + buffer_length_ = input.buffer_length(); + channel_count_ = input.channel_count(); + + signals_.resize(channel_count_); + strobes_.resize(channel_count_); + + centre_frequencies_.resize(channel_count_, 0.0f); + for (int i = 0; i < channel_count_; ++i) { + centre_frequencies_[i] = input.get_centre_frequency(i); + } + + for (int i = 0; i < channel_count_; ++i) { + signals_[i].resize(buffer_length_, 0.0f); + } + initialized_ = true; + return true; +} + +bool SignalBank::Validate() const { + if (sample_rate_ <= 0.0f) + return false; + + if (static_cast(signals_.size()) < 1) + return false; + + if (static_cast(signals_.size()) != channel_count_) + return false; + + if (static_cast(strobes_.size()) != channel_count_) + return false; + + for (int i = 0; i < channel_count_; ++i) { + if (static_cast(signals_[i].size()) != buffer_length_) + return false; + } + return true; +} + +inline const vector &SignalBank::operator[](int channel) const { + return signals_[channel]; +} + +inline float SignalBank::sample(int channel, int index) const { + return signals_[channel][index]; +} + +inline void SignalBank::set_sample(int channel, int index, float value) { + signals_[channel][index] = value; +} + +const deque &SignalBank::strobes(int channel) const { + return strobes_[channel]; +} + +float SignalBank::sample_rate() const { + return sample_rate_; +} + +int SignalBank::buffer_length() const { + return buffer_length_; +} + +int SignalBank::start_time() const { + return start_time_; +} + +void SignalBank::set_start_time(int start_time) { + start_time_ = start_time; +} + +float SignalBank::get_centre_frequency(int i) const { + if (i < channel_count_) + return centre_frequencies_[i]; + else + return 0.0f; +} + +void SignalBank::set_centre_frequency(int i, float cf) { + if (i < channel_count_) + centre_frequencies_[i] = cf; +} + +bool SignalBank::initialized() const { + return initialized_; +} + +int SignalBank::channel_count() const { + return channel_count_; +} +} diff -r 000000000000 -r 582cbe817f2c src/Support/SignalBank.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/Support/SignalBank.h Fri Feb 12 12:31:23 2010 +0000 @@ -0,0 +1,82 @@ +// Copyright 2010, Thomas Walters +// +// AIM-C: A C++ implementation of the Auditory Image Model +// http://www.acousticscale.org/AIMC +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with this program. If not, see . + +/*! \file + * \brief Basic data structure for a bank of signals with a common sample + * rate (in Hz), a centre frequency (in Hz) for each channel, a start time + * for the signal (in samples) and optionally, a set of strobe points in each + * channel (a vector of the indices of certain points). This is a struct + * rather than a class, and data members are accessed directly. Therefore is + * is up to the users of this structure to use it properly. Never assume that + * a SignalBank will be set up correctly when you receive it. A basic + * constructor and initialisation routines are provided. + */ + +/*! \author: Thomas Walters + * \date 2010/01/23 + * \version \$Id: SignalBank.h 4 2010-02-03 18:44:58Z tcw $ + */ + +#ifndef _AIMC_SUPPORT_SIGNALBANK_H_ +#define _AIMC_SUPPORT_SIGNALBANK_H_ + +#include +#include + +#include "Support/Common.h" + +namespace aimc { +using std::deque; +using std::vector; +class SignalBank { + public: + SignalBank(); + ~SignalBank(); + bool Initialize(int channel_count, int signal_length, float sample_rate); + + /* \brief Initialize the signal bank with the same parameters, buffer size + * and centre frequencies as the input signal bank + */ + bool Initialize(const SignalBank &input); + bool Validate() const; + inline const vector &operator[](int channel) const; + inline float sample(int channel, int index) const; + inline void set_sample(int channel, int index, float value); + const deque &strobes(int channel) const; + float sample_rate() const; + int buffer_length() const; + int start_time() const; + void set_start_time(int start_time); + float get_centre_frequency(int i) const; + void set_centre_frequency(int i, float cf); + bool initialized() const; + int channel_count() const; + private: + int channel_count_; + int buffer_length_; + vector > signals_; + vector > strobes_; + vector centre_frequencies_; + float sample_rate_; + int start_time_; + bool initialized_; + DISALLOW_COPY_AND_ASSIGN(SignalBank); +}; +} + +#endif // _AIMC_SUPPORT_SIGNALBANK_H_ diff -r 000000000000 -r 582cbe817f2c src/Support/SimpleIni.h --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/Support/SimpleIni.h Fri Feb 12 12:31:23 2010 +0000 @@ -0,0 +1,3258 @@ +/** @mainpage + + +
Library SimpleIni +
File SimpleIni.h +
Author Brodie Thiesfield [code at jellycan dot com] +
Source http://code.jellycan.com/simpleini/ +
Version 4.12 +
+ + Jump to the @link CSimpleIniTempl CSimpleIni @endlink interface documentation. + + @section intro INTRODUCTION + + This component allows an INI-style configuration file to be used on both + Windows and Linux/Unix. It is fast, simple and source code using this + component will compile unchanged on either OS. + + + @section features FEATURES + + - MIT Licence allows free use in all software (including GPL and commercial) + - multi-platform (Windows 95/98/ME/NT/2K/XP/2003, Windows CE, Linux, Unix) + - loading and saving of INI-style configuration files + - configuration files can have any newline format on all platforms + - liberal acceptance of file format + - key/values with no section + - removal of whitespace around sections, keys and values + - support for multi-line values (values with embedded newline characters) + - optional support for multiple keys with the same name + - optional case-insensitive sections and keys (for ASCII characters only) + - saves files with sections and keys in the same order as they were loaded + - preserves comments on the file, section and keys where possible. + - supports both char or wchar_t programming interfaces + - supports both MBCS (system locale) and UTF-8 file encodings + - system locale does not need to be UTF-8 on Linux/Unix to load UTF-8 file + - support for non-ASCII characters in section, keys, values and comments + - support for non-standard character types or file encodings + via user-written converter classes + - support for adding/modifying values programmatically + - compiles cleanly in the following compilers: + - Windows/VC6 (warning level 3) + - Windows/VC.NET 2003 (warning level 4) + - Windows/VC 2005 (warning level 4) + - Linux/gcc (-Wall) + + + @section usage USAGE SUMMARY + + -# Define the appropriate symbol for the converter you wish to use and + include the SimpleIni.h header file. If no specific converter is defined + then the default converter is used. The default conversion mode uses + SI_CONVERT_WIN32 on Windows and SI_CONVERT_GENERIC on all other + platforms. If you are using ICU then SI_CONVERT_ICU is supported on all + platforms. + -# Declare an instance the appropriate class. Note that the following + definitions are just shortcuts for commonly used types. Other types + (PRUnichar, unsigned short, unsigned char) are also possible. + +
Interface Case-sensitive Load UTF-8 Load MBCS Typedef +
SI_CONVERT_GENERIC +
char No Yes Yes #1 CSimpleIniA +
char Yes Yes Yes CSimpleIniCaseA +
wchar_t No Yes Yes CSimpleIniW +
wchar_t Yes Yes Yes CSimpleIniCaseW +
SI_CONVERT_WIN32 +
char No No #2 Yes CSimpleIniA +
char Yes Yes Yes CSimpleIniCaseA +
wchar_t No Yes Yes CSimpleIniW +
wchar_t Yes Yes Yes CSimpleIniCaseW +
SI_CONVERT_ICU +
char No Yes Yes CSimpleIniA +
char Yes Yes Yes CSimpleIniCaseA +
UChar No Yes Yes CSimpleIniW +
UChar Yes Yes Yes CSimpleIniCaseW +
+ #1 On Windows you are better to use CSimpleIniA with SI_CONVERT_WIN32.
+ #2 Only affects Windows. On Windows this uses MBCS functions and + so may fold case incorrectly leading to uncertain results. + -# Call Load() or LoadFile() to load and parse the INI configuration file + -# Access and modify the data of the file using the following functions + +
GetAllSections Return all section names +
GetAllKeys Return all key names within a section +
GetAllValues Return all values within a section & key +
GetSection Return all key names and values in a section +
GetSectionSize Return the number of keys in a section +
GetValue Return a value for a section & key +
SetValue Add or update a value for a section & key +
Delete Remove a section, or a key from a section +
+ -# Call Save() or SaveFile() to save the INI configuration data + + @section iostreams IO STREAMS + + SimpleIni supports reading from and writing to STL IO streams. Enable this + by defining SI_SUPPORT_IOSTREAMS before including the SimpleIni.h header + file. Ensure that if the streams are backed by a file (e.g. ifstream or + ofstream) then the flag ios_base::binary has been used when the file was + opened. + + @section multiline MULTI-LINE VALUES + + Values that span multiple lines are created using the following format. + + 
+        key = <<
+
+    Note the following:
+    - The text used for ENDTAG can be anything and is used to find
+      where the multi-line text ends.
+    - The newline after ENDTAG in the start tag, and the newline
+      before ENDTAG in the end tag is not included in the data value.
+    - The ending tag must be on it's own line with no whitespace before
+      or after it.
+    - The multi-line value is modified at load so that each line in the value
+      is delimited by a single '\\n' character on all platforms. At save time
+      it will be converted into the newline format used by the current
+      platform.
+
+    @section comments COMMENTS
+
+    Comments are preserved in the file within the following restrictions:
+    - Every file may have a single "file comment". It must start with the
+      first character in the file, and will end with the first non-comment
+      line in the file.
+    - Every section may have a single "section comment". It will start
+      with the first comment line following the file comment, or the last
+      data entry. It ends at the beginning of the section.
+    - Every key may have a single "key comment". This comment will start
+      with the first comment line following the section start, or the file
+      comment if there is no section name.
+    - Comments are set at the time that the file, section or key is first
+      created. The only way to modify a comment on a section or a key is to
+      delete that entry and recreate it with the new comment. There is no
+      way to change the file comment.
+
+    @section save SAVE ORDER
+
+    The sections and keys are written out in the same order as they were
+    read in from the file. Sections and keys added to the data after the
+    file has been loaded will be added to the end of the file when it is
+    written. There is no way to specify the location of a section or key
+    other than in first-created, first-saved order.
+
+    @section notes NOTES
+
+    - To load UTF-8 data on Windows 95, you need to use Microsoft Layer for
+      Unicode, or SI_CONVERT_GENERIC, or SI_CONVERT_ICU.
+    - When using SI_CONVERT_GENERIC, ConvertUTF.c must be compiled and linked.
+    - When using SI_CONVERT_ICU, ICU header files must be on the include
+      path and icuuc.lib must be linked in.
+    - To load a UTF-8 file on Windows AND expose it with SI_CHAR == char,
+      you should use SI_CONVERT_GENERIC.
+    - The collation (sorting) order used for sections and keys returned from
+      iterators is NOT DEFINED. If collation order of the text is important
+      then it should be done yourself by either supplying a replacement
+      SI_STRLESS class, or by sorting the strings external to this library.
+    - Usage of the  header on Windows can be disabled by defining
+      SI_NO_MBCS. This is defined automatically on Windows CE platforms.
+
+
+    @section licence MIT LICENCE
+
+    The licence text below is the boilerplate "MIT Licence" used from:
+    http://www.opensource.org/licenses/mit-license.php
+
+    Copyright (c) 2006-2008, Brodie Thiesfield
+
+    Permission is hereby granted, free of charge, to any person obtaining a copy
+    of this software and associated documentation files (the "Software"), to deal
+    in the Software without restriction, including without limitation the rights
+    to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+    copies of the Software, and to permit persons to whom the Software is furnished
+    to do so, subject to the following conditions:
+
+    The above copyright notice and this permission notice shall be included in
+    all copies or substantial portions of the Software.
+
+    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+    FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+    COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+    IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+    CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+*/
+
+#ifndef INCLUDED_SimpleIni_h
+#define INCLUDED_SimpleIni_h
+
+#if defined(_MSC_VER) && (_MSC_VER >= 1020)
+# pragma once
+#endif
+
+// Disable these warnings in MSVC:
+//  4127 "conditional expression is constant" as the conversion classes trigger
+//  it with the statement if (sizeof(SI_CHAR) == sizeof(char)). This test will
+//  be optimized away in a release build.
+//  4503 'insert' : decorated name length exceeded, name was truncated
+//  4702 "unreachable code" as the MS STL header causes it in release mode.
+//  Again, the code causing the warning will be cleaned up by the compiler.
+//  4786 "identifier truncated to 256 characters" as this is thrown hundreds
+//  of times VC6 as soon as STL is used.
+#ifdef _MSC_VER
+# pragma warning (push)
+# pragma warning (disable: 4127 4503 4702 4786)
+#endif
+
+#include 
+#include 
+#include 
+#include 
+#include 
+#include 
+
+#ifdef SI_SUPPORT_IOSTREAMS
+# include 
+#endif // SI_SUPPORT_IOSTREAMS
+
+#ifdef _DEBUG
+# ifndef assert
+#  include 
+# endif
+# define SI_ASSERT(x)   assert(x)
+#else
+# define SI_ASSERT(x)
+#endif
+
+enum SI_Error {
+    SI_OK       =  0,   //!< No error
+    SI_UPDATED  =  1,   //!< An existing value was updated
+    SI_INSERTED =  2,   //!< A new value was inserted
+
+    // note: test for any error with (retval < 0)
+    SI_FAIL     = -1,   //!< Generic failure
+    SI_NOMEM    = -2,   //!< Out of memory error
+    SI_FILE     = -3    //!< File error (see errno for detail error)
+};
+
+#define SI_UTF8_SIGNATURE     "\xEF\xBB\xBF"
+
+#ifdef _WIN32
+# define SI_NEWLINE_A   "\r\n"
+# define SI_NEWLINE_W   L"\r\n"
+#else // !_WIN32
+# define SI_NEWLINE_A   "\n"
+# define SI_NEWLINE_W   L"\n"
+#endif // _WIN32
+
+#if defined(SI_CONVERT_ICU)
+# include 
+#endif
+
+#if defined(_WIN32)
+# define SI_HAS_WIDE_FILE
+# define SI_WCHAR_T     wchar_t
+#elif defined(SI_CONVERT_ICU)
+# define SI_HAS_WIDE_FILE
+# define SI_WCHAR_T     UChar
+#endif
+
+
+// ---------------------------------------------------------------------------
+//                              MAIN TEMPLATE CLASS
+// ---------------------------------------------------------------------------
+
+/** Simple INI file reader.
+
+    This can be instantiated with the choice of unicode or native characterset,
+    and case sensitive or insensitive comparisons of section and key names.
+    The supported combinations are pre-defined with the following typedefs:
+
+    
+        
Interface   Case-sensitive  Typedef
+        
char        No              CSimpleIniA
+        
char        Yes             CSimpleIniCaseA
+        
wchar_t     No              CSimpleIniW
+        
wchar_t     Yes             CSimpleIniCaseW
+    

+
+    Note that using other types for the SI_CHAR is supported. For instance,
+    unsigned char, unsigned short, etc. Note that where the alternative type
+    is a different size to char/wchar_t you may need to supply new helper
+    classes for SI_STRLESS and SI_CONVERTER.
+ */
+template
+class CSimpleIniTempl
+{
+public:
+    /** key entry */
+    struct Entry {
+        const SI_CHAR * pItem;
+        const SI_CHAR * pComment;
+        int             nOrder;
+
+        Entry(const SI_CHAR * a_pszItem = NULL, int a_nOrder = 0)
+            : pItem(a_pszItem)
+            , pComment(NULL)
+            , nOrder(a_nOrder)
+        { }
+        Entry(const SI_CHAR * a_pszItem, const SI_CHAR * a_pszComment, int a_nOrder)
+            : pItem(a_pszItem)
+            , pComment(a_pszComment)
+            , nOrder(a_nOrder)
+        { }
+        Entry(const Entry & rhs) { operator=(rhs); }
+        Entry & operator=(const Entry & rhs) {
+            pItem    = rhs.pItem;
+            pComment = rhs.pComment;
+            nOrder   = rhs.nOrder;
+            return *this;
+        }
+
+#if defined(_MSC_VER) && _MSC_VER <= 1200
+        /** STL of VC6 doesn't allow me to specify my own comparator for list::sort() */
+        bool operator<(const Entry & rhs) const { return LoadOrder()(*this, rhs); }
+        bool operator>(const Entry & rhs) const { return LoadOrder()(rhs, *this); }
+#endif
+
+        /** Strict less ordering by name of key only */
+        struct KeyOrder : std::binary_function {
+            bool operator()(const Entry & lhs, const Entry & rhs) const {
+                const static SI_STRLESS isLess = SI_STRLESS();
+                return isLess(lhs.pItem, rhs.pItem);
+            }
+        };
+
+        /** Strict less ordering by order, and then name of key */
+        struct LoadOrder : std::binary_function {
+            bool operator()(const Entry & lhs, const Entry & rhs) const {
+                if (lhs.nOrder != rhs.nOrder) {
+                    return lhs.nOrder < rhs.nOrder;
+                }
+                return KeyOrder()(lhs.pItem, rhs.pItem);
+            }
+        };
+    };
+
+    /** map keys to values */
+    typedef std::multimap TKeyVal;
+
+    /** map sections to key/value map */
+    typedef std::map TSection;
+
+    /** set of dependent string pointers. Note that these pointers are
+        dependent on memory owned by CSimpleIni.
+    */
+    typedef std::list TNamesDepend;
+
+    /** interface definition for the OutputWriter object to pass to Save()
+        in order to output the INI file data.
+    */
+    class OutputWriter {
+    public:
+        OutputWriter() { }
+        virtual ~OutputWriter() { }
+        virtual void Write(const char * a_pBuf) = 0;
+    private:
+        OutputWriter(const OutputWriter &);             // disable
+        OutputWriter & operator=(const OutputWriter &); // disable
+    };
+
+    /** OutputWriter class to write the INI data to a file */
+    class FileWriter : public OutputWriter {
+        FILE * m_file;
+    public:
+        FileWriter(FILE * a_file) : m_file(a_file) { }
+        void Write(const char * a_pBuf) {
+            fputs(a_pBuf, m_file);
+        }
+    private:
+        FileWriter(const FileWriter &);             // disable
+        FileWriter & operator=(const FileWriter &); // disable
+    };
+
+    /** OutputWriter class to write the INI data to a string */
+    class StringWriter : public OutputWriter {
+        std::string & m_string;
+    public:
+        StringWriter(std::string & a_string) : m_string(a_string) { }
+        void Write(const char * a_pBuf) {
+            m_string.append(a_pBuf);
+        }
+    private:
+        StringWriter(const StringWriter &);             // disable
+        StringWriter & operator=(const StringWriter &); // disable
+    };
+
+#ifdef SI_SUPPORT_IOSTREAMS
+    /** OutputWriter class to write the INI data to an ostream */
+    class StreamWriter : public OutputWriter {
+        std::ostream & m_ostream;
+    public:
+        StreamWriter(std::ostream & a_ostream) : m_ostream(a_ostream) { }
+        void Write(const char * a_pBuf) {
+            m_ostream << a_pBuf;
+        }
+    private:
+        StreamWriter(const StreamWriter &);             // disable
+        StreamWriter & operator=(const StreamWriter &); // disable
+    };
+#endif // SI_SUPPORT_IOSTREAMS
+
+    /** Characterset conversion utility class to convert strings to the
+        same format as is used for the storage.
+    */
+    class Converter : private SI_CONVERTER {
+    public:
+        Converter(bool a_bStoreIsUtf8) : SI_CONVERTER(a_bStoreIsUtf8) {
+            m_scratch.resize(1024);
+        }
+        Converter(const Converter & rhs) { operator=(rhs); }
+        Converter & operator=(const Converter & rhs) {
+            m_scratch = rhs.m_scratch;
+            return *this;
+        }
+        bool ConvertToStore(const SI_CHAR * a_pszString) {
+            size_t uLen = SizeToStore(a_pszString);
+            if (uLen == (size_t)(-1)) {
+                return false;
+            }
+            while (uLen > m_scratch.size()) {
+                m_scratch.resize(m_scratch.size() * 2);
+            }
+            return SI_CONVERTER::ConvertToStore(
+                a_pszString,
+                const_cast(m_scratch.data()),
+                m_scratch.size());
+        }
+        const char * Data() { return m_scratch.data(); }
+    private:
+        std::string m_scratch;
+    };
+
+public:
+    /*-----------------------------------------------------------------------*/
+
+    /** Default constructor.
+
+        @param a_bIsUtf8     See the method SetUnicode() for details.
+        @param a_bMultiKey   See the method SetMultiKey() for details.
+        @param a_bMultiLine  See the method SetMultiLine() for details.
+     */
+    CSimpleIniTempl(
+        bool a_bIsUtf8    = false,
+        bool a_bMultiKey  = false,
+        bool a_bMultiLine = false
+        );
+
+    /** Destructor */
+    ~CSimpleIniTempl();
+
+    /** Deallocate all memory stored by this object */
+    void Reset();
+
+    /*-----------------------------------------------------------------------*/
+    /** @{ @name Settings */
+
+    /** Set the storage format of the INI data. This affects both the loading
+        and saving of the INI data using all of the Load/Save API functions.
+        This value cannot be changed after any INI data has been loaded.
+
+        If the file is not set to Unicode (UTF-8), then the data encoding is
+        assumed to be the OS native encoding. This encoding is the system
+        locale on Linux/Unix and the legacy MBCS encoding on Windows NT/2K/XP.
+        If the storage format is set to Unicode then the file will be loaded
+        as UTF-8 encoded data regardless of the native file encoding. If
+        SI_CHAR == char then all of the char* parameters take and return UTF-8
+        encoded data regardless of the system locale.
+
+        \param a_bIsUtf8     Assume UTF-8 encoding for the source?
+     */
+    void SetUnicode(bool a_bIsUtf8 = true) {
+        if (!m_pData) m_bStoreIsUtf8 = a_bIsUtf8;
+    }
+
+    /** Get the storage format of the INI data. */
+    bool IsUnicode() const { return m_bStoreIsUtf8; }
+
+    /** Should multiple identical keys be permitted in the file. If set to false
+        then the last value encountered will be used as the value of the key.
+        If set to true, then all values will be available to be queried. For
+        example, with the following input:
+
+        
+        [section]
+        test=value1
+        test=value2
+        

+
+        Then with SetMultiKey(true), both of the values "value1" and "value2"
+        will be returned for the key test. If SetMultiKey(false) is used, then
+        the value for "test" will only be "value2". This value may be changed
+        at any time.
+
+        \param a_bAllowMultiKey  Allow multi-keys in the source?
+     */
+    void SetMultiKey(bool a_bAllowMultiKey = true) {
+        m_bAllowMultiKey = a_bAllowMultiKey;
+    }
+
+    /** Get the storage format of the INI data. */
+    bool IsMultiKey() const { return m_bAllowMultiKey; }
+
+    /** Should data values be permitted to span multiple lines in the file. If
+        set to false then the multi-line construct <<
+            SI_CHAR     FORMAT
+            char        same format as when loaded (MBCS or UTF-8)
+            wchar_t     UTF-8
+            other       UTF-8
+        
+
+        Note that comments from the original data is preserved as per the
+        documentation on comments. The order of the sections and values
+        from the original file will be preserved.
+
+        Any data prepended or appended to the output device must use the the
+        same format (MBCS or UTF-8). You may use the GetConverter() method to
+        convert text to the correct format regardless of the output format
+        being used by SimpleIni.
+
+        To add a BOM to UTF-8 data, write it out manually at the very beginning
+        like is done in SaveFile when a_bUseBOM is true.
+
+        @param a_oOutput    Output writer to write the data to.
+
+        @param a_bAddSignature  Prepend the UTF-8 BOM if the output data is in
+                            UTF-8 format. If it is not UTF-8 then this value is
+                            ignored. Do not set this to true if anything has
+                            already been written to the OutputWriter.
+
+        @return SI_Error    See error definitions
+     */
+    SI_Error Save(
+        OutputWriter &  a_oOutput,
+        bool            a_bAddSignature = false
+        ) const;
+
+#ifdef SI_SUPPORT_IOSTREAMS
+    /** Save the INI data to an ostream. See Save() for details.
+
+        @param a_ostream    String to have the INI data appended to.
+
+        @param a_bAddSignature  Prepend the UTF-8 BOM if the output data is in
+                            UTF-8 format. If it is not UTF-8 then this value is
+                            ignored. Do not set this to true if anything has
+                            already been written to the stream.
+
+        @return SI_Error    See error definitions
+     */
+    SI_Error Save(
+        std::ostream &  a_ostream,
+        bool            a_bAddSignature = false
+        ) const
+    {
+        StreamWriter writer(a_ostream);
+        return Save(writer, a_bAddSignature);
+    }
+#endif // SI_SUPPORT_IOSTREAMS
+
+    /** Append the INI data to a string. See Save() for details.
+
+        @param a_sBuffer    String to have the INI data appended to.
+
+        @param a_bAddSignature  Prepend the UTF-8 BOM if the output data is in
+                            UTF-8 format. If it is not UTF-8 then this value is
+                            ignored. Do not set this to true if anything has
+                            already been written to the string.
+
+        @return SI_Error    See error definitions
+     */
+    SI_Error Save(
+        std::string &   a_sBuffer,
+        bool            a_bAddSignature = false
+        ) const
+    {
+        StringWriter writer(a_sBuffer);
+        return Save(writer, a_bAddSignature);
+    }
+
+    /*-----------------------------------------------------------------------*/
+    /** @}
+        @{ @name Accessing INI Data */
+
+    /** Retrieve all section names. The list is returned as an STL vector of
+        names and can be iterated or searched as necessary. Note that the
+        sort order of the returned strings is NOT DEFINED. You can sort
+        the names into the load order if desired. Search this file for ".sort"
+        for an example.
+
+        NOTE! This structure contains only pointers to strings. The actual
+        string data is stored in memory owned by CSimpleIni. Ensure that the
+        CSimpleIni object is not destroyed or Reset() while these pointers
+        are in use!
+
+        @param a_names          Vector that will receive all of the section
+                                 names. See note above!
+     */
+    void GetAllSections(
+        TNamesDepend & a_names
+        ) const;
+
+    /** Retrieve all unique key names in a section. The sort order of the
+        returned strings is NOT DEFINED. You can sort the names into the load 
+        order if desired. Search this file for ".sort" for an example. Only 
+        unique key names are returned.
+
+        NOTE! This structure contains only pointers to strings. The actual
+        string data is stored in memory owned by CSimpleIni. Ensure that the
+        CSimpleIni object is not destroyed or Reset() while these strings
+        are in use!
+
+        @param a_pSection       Section to request data for
+        @param a_names          List that will receive all of the key
+                                 names. See note above!
+
+        @return true            Section was found.
+        @return false           Matching section was not found.
+     */
+    bool GetAllKeys(
+        const SI_CHAR * a_pSection,
+        TNamesDepend &  a_names
+        ) const;
+
+    /** Retrieve all values for a specific key. This method can be used when
+        multiple keys are both enabled and disabled. Note that the sort order 
+        of the returned strings is NOT DEFINED. You can sort the names into 
+        the load order if desired. Search this file for ".sort" for an example.
+
+        NOTE! The returned values are pointers to string data stored in memory
+        owned by CSimpleIni. Ensure that the CSimpleIni object is not destroyed
+        or Reset while you are using this pointer!
+
+        @param a_pSection       Section to search
+        @param a_pKey           Key to search for
+        @param a_values         List to return if the key is not found
+
+        @return true            Key was found.
+        @return false           Matching section/key was not found.
+     */
+    bool GetAllValues(
+        const SI_CHAR * a_pSection,
+        const SI_CHAR * a_pKey,
+        TNamesDepend &  a_values
+        ) const;
+
+    /** Query the number of keys in a specific section. Note that if multiple
+        keys are enabled, then this value may be different to the number of
+        keys returned by GetAllKeys.
+
+        @param a_pSection       Section to request data for
+
+        @return -1              Section does not exist in the file
+        @return >=0             Number of keys in the section
+     */
+    int GetSectionSize(
+        const SI_CHAR * a_pSection
+        ) const;
+
+    /** Retrieve all key and value pairs for a section. The data is returned
+        as a pointer to an STL map and can be iterated or searched as
+        desired. Note that multiple entries for the same key may exist when
+        multiple keys have been enabled.
+
+        NOTE! This structure contains only pointers to strings. The actual
+        string data is stored in memory owned by CSimpleIni. Ensure that the
+        CSimpleIni object is not destroyed or Reset() while these strings
+        are in use!
+
+        @param a_pSection       Name of the section to return
+        @return boolean         Was a section matching the supplied
+                                name found.
+     */
+    const TKeyVal * GetSection(
+        const SI_CHAR * a_pSection
+        ) const;
+
+    /** Retrieve the value for a specific key. If multiple keys are enabled
+        (see SetMultiKey) then only the first value associated with that key
+        will be returned, see GetAllValues for getting all values with multikey.
+
+        NOTE! The returned value is a pointer to string data stored in memory
+        owned by CSimpleIni. Ensure that the CSimpleIni object is not destroyed
+        or Reset while you are using this pointer!
+
+        @param a_pSection       Section to search
+        @param a_pKey           Key to search for
+        @param a_pDefault       Value to return if the key is not found
+        @param a_pHasMultiple   Optionally receive notification of if there are
+                                multiple entries for this key.
+
+        @return a_pDefault      Key was not found in the section
+        @return other           Value of the key
+     */
+    const SI_CHAR * GetValue(
+        const SI_CHAR * a_pSection,
+        const SI_CHAR * a_pKey,
+        const SI_CHAR * a_pDefault     = NULL,
+        bool *          a_pHasMultiple = NULL
+        ) const;
+
+    /** Retrieve a numeric value for a specific key. If multiple keys are enabled
+        (see SetMultiKey) then only the first value associated with that key
+        will be returned, see GetAllValues for getting all values with multikey.
+
+        @param a_pSection       Section to search
+        @param a_pKey           Key to search for
+        @param a_nDefault       Value to return if the key is not found
+        @param a_pHasMultiple   Optionally receive notification of if there are
+                                multiple entries for this key.
+
+        @return a_nDefault      Key was not found in the section
+        @return other           Value of the key
+     */
+    long GetLongValue(
+        const SI_CHAR * a_pSection,
+        const SI_CHAR * a_pKey,
+        long            a_nDefault     = 0,
+        bool *          a_pHasMultiple = NULL
+        ) const;
+
+    /** Retrieve a boolean value for a specific key. If multiple keys are enabled
+        (see SetMultiKey) then only the first value associated with that key
+        will be returned, see GetAllValues for getting all values with multikey.
+
+        Strings starting with "t", "y", "on" or "1" are returned as logically true.
+        Strings starting with "f", "n", "of" or "0" are returned as logically false.
+        For all other values the default is returned. Character comparisons are 
+        case-insensitive.
+
+        @param a_pSection       Section to search
+        @param a_pKey           Key to search for
+        @param a_bDefault       Value to return if the key is not found
+        @param a_pHasMultiple   Optionally receive notification of if there are
+                                multiple entries for this key.
+
+        @return a_nDefault      Key was not found in the section
+        @return other           Value of the key
+     */
+    bool GetBoolValue(
+        const SI_CHAR * a_pSection,
+        const SI_CHAR * a_pKey,
+        bool            a_bDefault     = false,
+        bool *          a_pHasMultiple = NULL
+        ) const;
+
+    /** Add or update a section or value. This will always insert
+        when multiple keys are enabled.
+
+        @param a_pSection   Section to add or update
+        @param a_pKey       Key to add or update. Set to NULL to
+                            create an empty section.
+        @param a_pValue     Value to set. Set to NULL to create an
+                            empty section.
+        @param a_pComment   Comment to be associated with the section or the
+                            key. If a_pKey is NULL then it will be associated
+                            with the section, otherwise the key. Note that a
+                            comment may be set ONLY when the section or key is
+                            first created (i.e. when this function returns the
+                            value SI_INSERTED). If you wish to create a section
+                            with a comment then you need to create the section
+                            separately to the key. The comment string must be
+                            in full comment form already (have a comment
+                            character starting every line).
+        @param a_bForceReplace  Should all existing values in a multi-key INI
+                            file be replaced with this entry. This option has
+                            no effect if not using multi-key files. The 
+                            difference between Delete/SetValue and SetValue
+                            with a_bForceReplace = true, is that the load 
+                            order and comment will be preserved this way.
+
+        @return SI_Error    See error definitions
+        @return SI_UPDATED  Value was updated
+        @return SI_INSERTED Value was inserted
+     */
+    SI_Error SetValue(
+        const SI_CHAR * a_pSection,
+        const SI_CHAR * a_pKey,
+        const SI_CHAR * a_pValue,
+        const SI_CHAR * a_pComment      = NULL,
+        bool            a_bForceReplace = false
+        )
+    {
+        return AddEntry(a_pSection, a_pKey, a_pValue, a_pComment, a_bForceReplace, true);
+    }
+
+    /** Add or update a numeric value. This will always insert
+        when multiple keys are enabled.
+
+        @param a_pSection   Section to add or update
+        @param a_pKey       Key to add or update. 
+        @param a_nValue     Value to set. 
+        @param a_pComment   Comment to be associated with the key. See the 
+                            notes on SetValue() for comments.
+        @param a_bUseHex    By default the value will be written to the file 
+                            in decimal format. Set this to true to write it 
+                            as hexadecimal.
+        @param a_bForceReplace  Should all existing values in a multi-key INI
+                            file be replaced with this entry. This option has
+                            no effect if not using multi-key files. The 
+                            difference between Delete/SetLongValue and 
+                            SetLongValue with a_bForceReplace = true, is that 
+                            the load order and comment will be preserved this 
+                            way.
+
+        @return SI_Error    See error definitions
+        @return SI_UPDATED  Value was updated
+        @return SI_INSERTED Value was inserted
+     */
+    SI_Error SetLongValue(
+        const SI_CHAR * a_pSection,
+        const SI_CHAR * a_pKey,
+        long            a_nValue,
+        const SI_CHAR * a_pComment      = NULL,
+        bool            a_bUseHex       = false,
+        bool            a_bForceReplace = false
+        );
+
+    /** Add or update a boolean value. This will always insert
+        when multiple keys are enabled.
+
+        @param a_pSection   Section to add or update
+        @param a_pKey       Key to add or update. 
+        @param a_bValue     Value to set. 
+        @param a_pComment   Comment to be associated with the key. See the 
+                            notes on SetValue() for comments.
+        @param a_bForceReplace  Should all existing values in a multi-key INI
+                            file be replaced with this entry. This option has
+                            no effect if not using multi-key files. The 
+                            difference between Delete/SetBoolValue and 
+                            SetBoolValue with a_bForceReplace = true, is that 
+                            the load order and comment will be preserved this 
+                            way.
+
+        @return SI_Error    See error definitions
+        @return SI_UPDATED  Value was updated
+        @return SI_INSERTED Value was inserted
+     */
+    SI_Error SetBoolValue(
+        const SI_CHAR * a_pSection,
+        const SI_CHAR * a_pKey,
+        bool            a_bValue,
+        const SI_CHAR * a_pComment      = NULL,
+        bool            a_bForceReplace = false
+        );
+
+    /** Delete an entire section, or a key from a section. Note that the
+        data returned by GetSection is invalid and must not be used after
+        anything has been deleted from that section using this method.
+        Note when multiple keys is enabled, this will delete all keys with
+        that name; there is no way to selectively delete individual key/values
+        in this situation.
+
+        @param a_pSection       Section to delete key from, or if
+                                a_pKey is NULL, the section to remove.
+        @param a_pKey           Key to remove from the section. Set to
+                                NULL to remove the entire section.
+        @param a_bRemoveEmpty   If the section is empty after this key has
+                                been deleted, should the empty section be
+                                removed?
+
+        @return true            Key or section was deleted.
+        @return false           Key or section was not found.
+     */
+    bool Delete(
+        const SI_CHAR * a_pSection,
+        const SI_CHAR * a_pKey,
+        bool            a_bRemoveEmpty = false
+        );
+
+    /*-----------------------------------------------------------------------*/
+    /** @}
+        @{ @name Converter */
+
+    /** Return a conversion object to convert text to the same encoding
+        as is used by the Save(), SaveFile() and SaveString() functions.
+        Use this to prepare the strings that you wish to append or prepend
+        to the output INI data.
+     */
+    Converter GetConverter() const {
+        return Converter(m_bStoreIsUtf8);
+    }
+
+    /*-----------------------------------------------------------------------*/
+    /** @} */
+
+private:
+    // copying is not permitted
+    CSimpleIniTempl(const CSimpleIniTempl &); // disabled
+    CSimpleIniTempl & operator=(const CSimpleIniTempl &); // disabled
+
+    /** Parse the data looking for a file comment and store it if found.
+    */
+    SI_Error FindFileComment(
+        SI_CHAR *&      a_pData,
+        bool            a_bCopyStrings
+        );
+
+    /** Parse the data looking for the next valid entry. The memory pointed to
+        by a_pData is modified by inserting NULL characters. The pointer is
+        updated to the current location in the block of text.
+    */
+    bool FindEntry(
+        SI_CHAR *&  a_pData,
+        const SI_CHAR *&  a_pSection,
+        const SI_CHAR *&  a_pKey,
+        const SI_CHAR *&  a_pVal,
+        const SI_CHAR *&  a_pComment
+        ) const;
+
+    /** Add the section/key/value to our data.
+
+        @param a_pSection   Section name. Sections will be created if they
+                            don't already exist.
+        @param a_pKey       Key name. May be NULL to create an empty section.
+                            Existing entries will be updated. New entries will
+                            be created.
+        @param a_pValue     Value for the key.
+        @param a_pComment   Comment to be associated with the section or the
+                            key. If a_pKey is NULL then it will be associated
+                            with the section, otherwise the key. This must be
+                            a string in full comment form already (have a
+                            comment character starting every line).
+        @param a_bForceReplace  Should all existing values in a multi-key INI
+                            file be replaced with this entry. This option has
+                            no effect if not using multi-key files. The 
+                            difference between Delete/AddEntry and AddEntry
+                            with a_bForceReplace = true, is that the load 
+                            order and comment will be preserved this way.
+        @param a_bCopyStrings   Should copies of the strings be made or not.
+                            If false then the pointers will be used as is.
+    */
+    SI_Error AddEntry(
+        const SI_CHAR * a_pSection,
+        const SI_CHAR * a_pKey,
+        const SI_CHAR * a_pValue,
+        const SI_CHAR * a_pComment,
+        bool            a_bForceReplace,
+        bool            a_bCopyStrings
+        );
+
+    /** Is the supplied character a whitespace character? */
+    inline bool IsSpace(SI_CHAR ch) const {
+        return (ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n');
+    }
+
+    /** Does the supplied character start a comment line? */
+    inline bool IsComment(SI_CHAR ch) const {
+        return (ch == ';' || ch == '#');
+    }
+
+
+    /** Skip over a newline character (or characters) for either DOS or UNIX */
+    inline void SkipNewLine(SI_CHAR *& a_pData) const {
+        a_pData += (*a_pData == '\r' && *(a_pData+1) == '\n') ? 2 : 1;
+    }
+
+    /** Make a copy of the supplied string, replacing the original pointer */
+    SI_Error CopyString(const SI_CHAR *& a_pString);
+
+    /** Delete a string from the copied strings buffer if necessary */
+    void DeleteString(const SI_CHAR * a_pString);
+
+    /** Internal use of our string comparison function */
+    bool IsLess(const SI_CHAR * a_pLeft, const SI_CHAR * a_pRight) const {
+        const static SI_STRLESS isLess = SI_STRLESS();
+        return isLess(a_pLeft, a_pRight);
+    }
+
+    bool IsMultiLineTag(const SI_CHAR * a_pData) const;
+    bool IsMultiLineData(const SI_CHAR * a_pData) const;
+    bool LoadMultiLineText(
+        SI_CHAR *&          a_pData,
+        const SI_CHAR *&    a_pVal,
+        const SI_CHAR *     a_pTagName,
+        bool                a_bAllowBlankLinesInComment = false
+        ) const;
+    bool IsNewLineChar(SI_CHAR a_c) const;
+
+    bool OutputMultiLineText(
+        OutputWriter &  a_oOutput,
+        Converter &     a_oConverter,
+        const SI_CHAR * a_pText
+        ) const;
+
+private:
+    /** Copy of the INI file data in our character format. This will be
+        modified when parsed to have NULL characters added after all
+        interesting string entries. All of the string pointers to sections,
+        keys and values point into this block of memory.
+     */
+    SI_CHAR * m_pData;
+
+    /** Length of the data that we have stored. Used when deleting strings
+        to determine if the string is stored here or in the allocated string
+        buffer.
+     */
+    size_t m_uDataLen;
+
+    /** File comment for this data, if one exists. */
+    const SI_CHAR * m_pFileComment;
+
+    /** Parsed INI data. Section -> (Key -> Value). */
+    TSection m_data;
+
+    /** This vector stores allocated memory for copies of strings that have
+        been supplied after the file load. It will be empty unless SetValue()
+        has been called.
+     */
+    TNamesDepend m_strings;
+
+    /** Is the format of our datafile UTF-8 or MBCS? */
+    bool m_bStoreIsUtf8;
+
+    /** Are multiple values permitted for the same key? */
+    bool m_bAllowMultiKey;
+
+    /** Are data values permitted to span multiple lines? */
+    bool m_bAllowMultiLine;
+
+    /** Should spaces be written out surrounding the equals sign? */
+    bool m_bSpaces;
+    
+    /** Next order value, used to ensure sections and keys are output in the
+        same order that they are loaded/added.
+     */
+    int m_nOrder;
+};
+
+// ---------------------------------------------------------------------------
+//                                  IMPLEMENTATION
+// ---------------------------------------------------------------------------
+
+template
+CSimpleIniTempl::CSimpleIniTempl(
+    bool a_bIsUtf8,
+    bool a_bAllowMultiKey,
+    bool a_bAllowMultiLine
+    )
+  : m_pData(0)
+  , m_uDataLen(0)
+  , m_pFileComment(NULL)
+  , m_bStoreIsUtf8(a_bIsUtf8)
+  , m_bAllowMultiKey(a_bAllowMultiKey)
+  , m_bAllowMultiLine(a_bAllowMultiLine)
+  , m_bSpaces(true)
+  , m_nOrder(0)
+{ }
+
+template
+CSimpleIniTempl::~CSimpleIniTempl()
+{
+    Reset();
+}
+
+template
+void
+CSimpleIniTempl::Reset()
+{
+    // remove all data
+    delete[] m_pData;
+    m_pData = NULL;
+    m_uDataLen = 0;
+    m_pFileComment = NULL;
+    if (!m_data.empty()) {
+        m_data.erase(m_data.begin(), m_data.end());
+    }
+
+    // remove all strings
+    if (!m_strings.empty()) {
+        typename TNamesDepend::iterator i = m_strings.begin();
+        for (; i != m_strings.end(); ++i) {
+            delete[] const_cast(i->pItem);
+        }
+        m_strings.erase(m_strings.begin(), m_strings.end());
+    }
+}
+
+template
+SI_Error
+CSimpleIniTempl::LoadFile(
+    const char * a_pszFile
+    )
+{
+    FILE * fp = NULL;
+#if __STDC_WANT_SECURE_LIB__ && !_WIN32_WCE
+    fopen_s(&fp, a_pszFile, "rb");
+#else // !__STDC_WANT_SECURE_LIB__
+    fp = fopen(a_pszFile, "rb");
+#endif // __STDC_WANT_SECURE_LIB__
+    if (!fp) {
+        return SI_FILE;
+    }
+    SI_Error rc = LoadFile(fp);
+    fclose(fp);
+    return rc;
+}
+
+#ifdef SI_HAS_WIDE_FILE
+template
+SI_Error
+CSimpleIniTempl::LoadFile(
+    const SI_WCHAR_T * a_pwszFile
+    )
+{
+#ifdef _WIN32
+    FILE * fp = NULL;
+#if __STDC_WANT_SECURE_LIB__ && !_WIN32_WCE
+    _wfopen_s(&fp, a_pwszFile, L"rb");
+#else // !__STDC_WANT_SECURE_LIB__
+    fp = _wfopen(a_pwszFile, L"rb");
+#endif // __STDC_WANT_SECURE_LIB__
+    if (!fp) return SI_FILE;
+    SI_Error rc = LoadFile(fp);
+    fclose(fp);
+    return rc;
+#else // !_WIN32 (therefore SI_CONVERT_ICU)
+    char szFile[256];
+    u_austrncpy(szFile, a_pwszFile, sizeof(szFile));
+    return LoadFile(szFile);
+#endif // _WIN32
+}
+#endif // SI_HAS_WIDE_FILE
+
+template
+SI_Error
+CSimpleIniTempl::LoadFile(
+    FILE * a_fpFile
+    )
+{
+    // load the raw file data
+    int retval = fseek(a_fpFile, 0, SEEK_END);
+    if (retval != 0) {
+        return SI_FILE;
+    }
+    long lSize = ftell(a_fpFile);
+    if (lSize < 0) {
+        return SI_FILE;
+    }
+    if (lSize == 0) {
+        return SI_OK;
+    }
+    char * pData = new char[lSize];
+    if (!pData) {
+        return SI_NOMEM;
+    }
+    fseek(a_fpFile, 0, SEEK_SET);
+    size_t uRead = fread(pData, sizeof(char), lSize, a_fpFile);
+    if (uRead != (size_t) lSize) {
+        delete[] pData;
+        return SI_FILE;
+    }
+
+    // convert the raw data to unicode
+    SI_Error rc = Load(pData, uRead);
+    delete[] pData;
+    return rc;
+}
+
+template
+SI_Error
+CSimpleIniTempl::Load(
+    const char *    a_pData,
+    size_t          a_uDataLen
+    )
+{
+    SI_CONVERTER converter(m_bStoreIsUtf8);
+
+    if (a_uDataLen == 0) {
+        return SI_OK;
+    }
+
+    // consume the UTF-8 BOM if it exists
+    if (m_bStoreIsUtf8 && a_uDataLen >= 3) {
+        if (memcmp(a_pData, SI_UTF8_SIGNATURE, 3) == 0) {
+            a_pData    += 3;
+            a_uDataLen -= 3;
+        }
+    }
+
+    // determine the length of the converted data
+    size_t uLen = converter.SizeFromStore(a_pData, a_uDataLen);
+    if (uLen == (size_t)(-1)) {
+        return SI_FAIL;
+    }
+
+    // allocate memory for the data, ensure that there is a NULL
+    // terminator wherever the converted data ends
+    SI_CHAR * pData = new SI_CHAR[uLen+1];
+    if (!pData) {
+        return SI_NOMEM;
+    }
+    memset(pData, 0, sizeof(SI_CHAR)*(uLen+1));
+
+    // convert the data
+    if (!converter.ConvertFromStore(a_pData, a_uDataLen, pData, uLen)) {
+        delete[] pData;
+        return SI_FAIL;
+    }
+
+    // parse it
+    const static SI_CHAR empty = 0;
+    SI_CHAR * pWork = pData;
+    const SI_CHAR * pSection = ∅
+    const SI_CHAR * pItem = NULL;
+    const SI_CHAR * pVal = NULL;
+    const SI_CHAR * pComment = NULL;
+
+    // We copy the strings if we are loading data into this class when we
+    // already have stored some.
+    bool bCopyStrings = (m_pData != NULL);
+
+    // find a file comment if it exists, this is a comment that starts at the
+    // beginning of the file and continues until the first blank line.
+    SI_Error rc = FindFileComment(pWork, bCopyStrings);
+    if (rc < 0) return rc;
+
+    // add every entry in the file to the data table
+    while (FindEntry(pWork, pSection, pItem, pVal, pComment)) {
+        rc = AddEntry(pSection, pItem, pVal, pComment, false, bCopyStrings);
+        if (rc < 0) return rc;
+    }
+
+    // store these strings if we didn't copy them
+    if (bCopyStrings) {
+        delete[] pData;
+    }
+    else {
+        m_pData = pData;
+        m_uDataLen = uLen+1;
+    }
+
+    return SI_OK;
+}
+
+#ifdef SI_SUPPORT_IOSTREAMS
+template
+SI_Error
+CSimpleIniTempl::Load(
+    std::istream & a_istream
+    )
+{
+    std::string strData;
+    char szBuf[512];
+    do {
+        a_istream.get(szBuf, sizeof(szBuf), '\0');
+        strData.append(szBuf);
+    }
+    while (a_istream.good());
+    return Load(strData);
+}
+#endif // SI_SUPPORT_IOSTREAMS
+
+template
+SI_Error
+CSimpleIniTempl::FindFileComment(
+    SI_CHAR *&      a_pData,
+    bool            a_bCopyStrings
+    )
+{
+    // there can only be a single file comment
+    if (m_pFileComment) {
+        return SI_OK;
+    }
+
+    // Load the file comment as multi-line text, this will modify all of
+    // the newline characters to be single \n chars
+    if (!LoadMultiLineText(a_pData, m_pFileComment, NULL, false)) {
+        return SI_OK;
+    }
+
+    // copy the string if necessary
+    if (a_bCopyStrings) {
+        SI_Error rc = CopyString(m_pFileComment);
+        if (rc < 0) return rc;
+    }
+
+    return SI_OK;
+}
+
+template
+bool
+CSimpleIniTempl::FindEntry(
+    SI_CHAR *&        a_pData,
+    const SI_CHAR *&  a_pSection,
+    const SI_CHAR *&  a_pKey,
+    const SI_CHAR *&  a_pVal,
+    const SI_CHAR *&  a_pComment
+    ) const
+{
+    a_pComment = NULL;
+
+    SI_CHAR * pTrail = NULL;
+    while (*a_pData) {
+        // skip spaces and empty lines
+        while (*a_pData && IsSpace(*a_pData)) {
+            ++a_pData;
+        }
+        if (!*a_pData) {
+            break;
+        }
+
+        // skip processing of comment lines but keep a pointer to
+        // the start of the comment.
+        if (IsComment(*a_pData)) {
+            LoadMultiLineText(a_pData, a_pComment, NULL, true);
+            continue;
+        }
+
+        // process section names
+        if (*a_pData == '[') {
+            // skip leading spaces
+            ++a_pData;
+            while (*a_pData && IsSpace(*a_pData)) {
+                ++a_pData;
+            }
+
+            // find the end of the section name (it may contain spaces)
+            // and convert it to lowercase as necessary
+            a_pSection = a_pData;
+            while (*a_pData && *a_pData != ']' && !IsNewLineChar(*a_pData)) {
+                ++a_pData;
+            }
+
+            // if it's an invalid line, just skip it
+            if (*a_pData != ']') {
+                continue;
+            }
+
+            // remove trailing spaces from the section
+            pTrail = a_pData - 1;
+            while (pTrail >= a_pSection && IsSpace(*pTrail)) {
+                --pTrail;
+            }
+            ++pTrail;
+            *pTrail = 0;
+
+            // skip to the end of the line
+            ++a_pData;  // safe as checked that it == ']' above
+            while (*a_pData && !IsNewLineChar(*a_pData)) {
+                ++a_pData;
+            }
+
+            a_pKey = NULL;
+            a_pVal = NULL;
+            return true;
+        }
+
+        // find the end of the key name (it may contain spaces)
+        // and convert it to lowercase as necessary
+        a_pKey = a_pData;
+        while (*a_pData && *a_pData != '=' && !IsNewLineChar(*a_pData)) {
+            ++a_pData;
+        }
+
+        // if it's an invalid line, just skip it
+        if (*a_pData != '=') {
+            continue;
+        }
+
+        // empty keys are invalid
+        if (a_pKey == a_pData) {
+            while (*a_pData && !IsNewLineChar(*a_pData)) {
+                ++a_pData;
+            }
+            continue;
+        }
+
+        // remove trailing spaces from the key
+        pTrail = a_pData - 1;
+        while (pTrail >= a_pKey && IsSpace(*pTrail)) {
+            --pTrail;
+        }
+        ++pTrail;
+        *pTrail = 0;
+
+        // skip leading whitespace on the value
+        ++a_pData;  // safe as checked that it == '=' above
+        while (*a_pData && !IsNewLineChar(*a_pData) && IsSpace(*a_pData)) {
+            ++a_pData;
+        }
+
+        // find the end of the value which is the end of this line
+        a_pVal = a_pData;
+        while (*a_pData && !IsNewLineChar(*a_pData)) {
+            ++a_pData;
+        }
+
+        // remove trailing spaces from the value
+        pTrail = a_pData - 1;
+        if (*a_pData) { // prepare for the next round
+            SkipNewLine(a_pData);
+        }
+        while (pTrail >= a_pVal && IsSpace(*pTrail)) {
+            --pTrail;
+        }
+        ++pTrail;
+        *pTrail = 0;
+
+        // check for multi-line entries
+        if (m_bAllowMultiLine && IsMultiLineTag(a_pVal)) {
+            // skip the "<<<" to get the tag that will end the multiline
+            const SI_CHAR * pTagName = a_pVal + 3;
+            return LoadMultiLineText(a_pData, a_pVal, pTagName);
+        }
+
+        // return the standard entry
+        return true;
+    }
+
+    return false;
+}
+
+template
+bool
+CSimpleIniTempl::IsMultiLineTag(
+    const SI_CHAR * a_pVal
+    ) const
+{
+    // check for the "<<<" prefix for a multi-line entry
+    if (*a_pVal++ != '<') return false;
+    if (*a_pVal++ != '<') return false;
+    if (*a_pVal++ != '<') return false;
+    return true;
+}
+
+template
+bool
+CSimpleIniTempl::IsMultiLineData(
+    const SI_CHAR * a_pData
+    ) const
+{
+    // data is multi-line if it has any of the following features:
+    //  * whitespace prefix
+    //  * embedded newlines
+    //  * whitespace suffix
+
+    // empty string
+    if (!*a_pData) {
+        return false;
+    }
+
+    // check for prefix
+    if (IsSpace(*a_pData)) {
+        return true;
+    }
+
+    // embedded newlines
+    while (*a_pData) {
+        if (IsNewLineChar(*a_pData)) {
+            return true;
+        }
+        ++a_pData;
+    }
+
+    // check for suffix
+    if (IsSpace(*--a_pData)) {
+        return true;
+    }
+
+    return false;
+}
+
+template
+bool
+CSimpleIniTempl::IsNewLineChar(
+    SI_CHAR a_c
+    ) const
+{
+    return (a_c == '\n' || a_c == '\r');
+}
+
+template
+bool
+CSimpleIniTempl::LoadMultiLineText(
+    SI_CHAR *&          a_pData,
+    const SI_CHAR *&    a_pVal,
+    const SI_CHAR *     a_pTagName,
+    bool                a_bAllowBlankLinesInComment
+    ) const
+{
+    // we modify this data to strip all newlines down to a single '\n'
+    // character. This means that on Windows we need to strip out some
+    // characters which will make the data shorter.
+    // i.e.  LINE1-LINE1\r\nLINE2-LINE2\0 will become
+    //       LINE1-LINE1\nLINE2-LINE2\0
+    // The pDataLine entry is the pointer to the location in memory that
+    // the current line needs to start to run following the existing one.
+    // This may be the same as pCurrLine in which case no move is needed.
+    SI_CHAR * pDataLine = a_pData;
+    SI_CHAR * pCurrLine;
+
+    // value starts at the current line
+    a_pVal = a_pData;
+
+    // find the end tag. This tag must start in column 1 and be
+    // followed by a newline. No whitespace removal is done while
+    // searching for this tag.
+    SI_CHAR cEndOfLineChar = *a_pData;
+    for(;;) {
+        // if we are loading comments then we need a comment character as
+        // the first character on every line
+        if (!a_pTagName && !IsComment(*a_pData)) {
+            // if we aren't allowing blank lines then we're done
+            if (!a_bAllowBlankLinesInComment) {
+                break;
+            }
+
+            // if we are allowing blank lines then we only include them
+            // in this comment if another comment follows, so read ahead
+            // to find out.
+            SI_CHAR * pCurr = a_pData;
+            int nNewLines = 0;
+            while (IsSpace(*pCurr)) {
+                if (IsNewLineChar(*pCurr)) {
+                    ++nNewLines;
+                    SkipNewLine(pCurr);
+                }
+                else {
+                    ++pCurr;
+                }
+            }
+
+            // we have a comment, add the blank lines to the output
+            // and continue processing from here
+            if (IsComment(*pCurr)) {
+                for (; nNewLines > 0; --nNewLines) *pDataLine++ = '\n';
+                a_pData = pCurr;
+                continue;
+            }
+
+            // the comment ends here
+            break;
+        }
+
+        // find the end of this line
+        pCurrLine = a_pData;
+        while (*a_pData && !IsNewLineChar(*a_pData)) ++a_pData;
+
+        // move this line down to the location that it should be if necessary
+        if (pDataLine < pCurrLine) {
+            size_t nLen = (size_t) (a_pData - pCurrLine);
+            memmove(pDataLine, pCurrLine, nLen * sizeof(SI_CHAR));
+            pDataLine[nLen] = '\0';
+        }
+
+        // end the line with a NULL
+        cEndOfLineChar = *a_pData;
+        *a_pData = 0;
+
+        // if are looking for a tag then do the check now. This is done before
+        // checking for end of the data, so that if we have the tag at the end
+        // of the data then the tag is removed correctly.
+        if (a_pTagName &&
+            (!IsLess(pDataLine, a_pTagName) && !IsLess(a_pTagName, pDataLine)))
+        {
+            break;
+        }
+
+        // if we are at the end of the data then we just automatically end
+        // this entry and return the current data.
+        if (!cEndOfLineChar) {
+            return true;
+        }
+
+        // otherwise we need to process this newline to ensure that it consists
+        // of just a single \n character.
+        pDataLine += (a_pData - pCurrLine);
+        *a_pData = cEndOfLineChar;
+        SkipNewLine(a_pData);
+        *pDataLine++ = '\n';
+    }
+
+    // if we didn't find a comment at all then return false
+    if (a_pVal == a_pData) {
+        a_pVal = NULL;
+        return false;
+    }
+
+    // the data (which ends at the end of the last line) needs to be
+    // null-terminated BEFORE before the newline character(s). If the
+    // user wants a new line in the multi-line data then they need to
+    // add an empty line before the tag.
+    *--pDataLine = '\0';
+
+    // if looking for a tag and if we aren't at the end of the data,
+    // then move a_pData to the start of the next line.
+    if (a_pTagName && cEndOfLineChar) {
+        SI_ASSERT(IsNewLineChar(cEndOfLineChar));
+        *a_pData = cEndOfLineChar;
+        SkipNewLine(a_pData);
+    }
+
+    return true;
+}
+
+template
+SI_Error
+CSimpleIniTempl::CopyString(
+    const SI_CHAR *& a_pString
+    )
+{
+    size_t uLen = 0;
+    if (sizeof(SI_CHAR) == sizeof(char)) {
+        uLen = strlen((const char *)a_pString);
+    }
+    else if (sizeof(SI_CHAR) == sizeof(wchar_t)) {
+        uLen = wcslen((const wchar_t *)a_pString);
+    }
+    else {
+        for ( ; a_pString[uLen]; ++uLen) /*loop*/ ;
+    }
+    ++uLen; // NULL character
+    SI_CHAR * pCopy = new SI_CHAR[uLen];
+    if (!pCopy) {
+        return SI_NOMEM;
+    }
+    memcpy(pCopy, a_pString, sizeof(SI_CHAR)*uLen);
+    m_strings.push_back(pCopy);
+    a_pString = pCopy;
+    return SI_OK;
+}
+
+template
+SI_Error
+CSimpleIniTempl::AddEntry(
+    const SI_CHAR * a_pSection,
+    const SI_CHAR * a_pKey,
+    const SI_CHAR * a_pValue,
+    const SI_CHAR * a_pComment,
+    bool            a_bForceReplace,
+    bool            a_bCopyStrings
+    )
+{
+    SI_Error rc;
+    bool bInserted = false;
+
+    SI_ASSERT(!a_pComment || IsComment(*a_pComment));
+
+    // if we are copying strings then make a copy of the comment now
+    // because we will need it when we add the entry.
+    if (a_bCopyStrings && a_pComment) {
+        rc = CopyString(a_pComment);
+        if (rc < 0) return rc;
+    }
+
+    // create the section entry if necessary
+    typename TSection::iterator iSection = m_data.find(a_pSection);
+    if (iSection == m_data.end()) {
+        // if the section doesn't exist then we need a copy as the
+        // string needs to last beyond the end of this function
+        if (a_bCopyStrings) {
+            rc = CopyString(a_pSection);
+            if (rc < 0) return rc;
+        }
+
+        // only set the comment if this is a section only entry
+        Entry oSection(a_pSection, ++m_nOrder);
+        if (a_pComment && (!a_pKey || !a_pValue)) {
+            oSection.pComment = a_pComment;
+        }
+
+        typename TSection::value_type oEntry(oSection, TKeyVal());
+        typedef typename TSection::iterator SectionIterator;
+        std::pair i = m_data.insert(oEntry);
+        iSection = i.first;
+        bInserted = true;
+    }
+    if (!a_pKey || !a_pValue) {
+        // section only entries are specified with pItem and pVal as NULL
+        return bInserted ? SI_INSERTED : SI_UPDATED;
+    }
+
+    // check for existence of the key
+    TKeyVal & keyval = iSection->second;
+    typename TKeyVal::iterator iKey = keyval.find(a_pKey);
+
+    // remove all existing entries but save the load order and
+    // comment of the first entry
+    int nLoadOrder = ++m_nOrder;
+    if (iKey != keyval.end() && m_bAllowMultiKey && a_bForceReplace) {
+        const SI_CHAR * pComment = NULL;
+        while (iKey != keyval.end() && !IsLess(a_pKey, iKey->first.pItem)) {
+            if (iKey->first.nOrder < nLoadOrder) {
+                nLoadOrder = iKey->first.nOrder;
+                pComment   = iKey->first.pComment;
+            }
+            ++iKey;
+        }
+        if (pComment) {
+            DeleteString(a_pComment);
+            a_pComment = pComment;
+            CopyString(a_pComment);
+        }
+        Delete(a_pSection, a_pKey);
+        iKey = keyval.end();
+    }
+
+    // make string copies if necessary
+    bool bForceCreateNewKey = m_bAllowMultiKey && !a_bForceReplace;
+    if (a_bCopyStrings) {
+        if (bForceCreateNewKey || iKey == keyval.end()) {
+            // if the key doesn't exist then we need a copy as the
+            // string needs to last beyond the end of this function
+            // because we will be inserting the key next
+            rc = CopyString(a_pKey);
+            if (rc < 0) return rc;
+        }
+
+        // we always need a copy of the value
+        rc = CopyString(a_pValue);
+        if (rc < 0) return rc;
+    }
+
+    // create the key entry
+    if (iKey == keyval.end() || bForceCreateNewKey) {
+        Entry oKey(a_pKey, nLoadOrder);
+        if (a_pComment) {
+            oKey.pComment = a_pComment;
+        }
+        typename TKeyVal::value_type oEntry(oKey, NULL);
+        iKey = keyval.insert(oEntry);
+        bInserted = true;
+    }
+    iKey->second = a_pValue;
+    return bInserted ? SI_INSERTED : SI_UPDATED;
+}
+
+template
+const SI_CHAR *
+CSimpleIniTempl::GetValue(
+    const SI_CHAR * a_pSection,
+    const SI_CHAR * a_pKey,
+    const SI_CHAR * a_pDefault,
+    bool *          a_pHasMultiple
+    ) const
+{
+    if (a_pHasMultiple) {
+        *a_pHasMultiple = false;
+    }
+    if (!a_pSection || !a_pKey) {
+        return a_pDefault;
+    }
+    typename TSection::const_iterator iSection = m_data.find(a_pSection);
+    if (iSection == m_data.end()) {
+        return a_pDefault;
+    }
+    typename TKeyVal::const_iterator iKeyVal = iSection->second.find(a_pKey);
+    if (iKeyVal == iSection->second.end()) {
+        return a_pDefault;
+    }
+
+    // check for multiple entries with the same key
+    if (m_bAllowMultiKey && a_pHasMultiple) {
+        typename TKeyVal::const_iterator iTemp = iKeyVal;
+        if (++iTemp != iSection->second.end()) {
+            if (!IsLess(a_pKey, iTemp->first.pItem)) {
+                *a_pHasMultiple = true;
+            }
+        }
+    }
+
+    return iKeyVal->second;
+}
+
+template
+long
+CSimpleIniTempl::GetLongValue(
+    const SI_CHAR * a_pSection,
+    const SI_CHAR * a_pKey,
+    long            a_nDefault,
+    bool *          a_pHasMultiple
+    ) const
+{
+    // return the default if we don't have a value
+    const SI_CHAR * pszValue = GetValue(a_pSection, a_pKey, NULL, a_pHasMultiple);
+    if (!pszValue || !*pszValue) return a_nDefault;
+
+    // convert to UTF-8/MBCS which for a numeric value will be the same as ASCII
+    char szValue[64] = { 0 };
+    SI_CONVERTER c(m_bStoreIsUtf8);
+    if (!c.ConvertToStore(pszValue, szValue, sizeof(szValue))) {
+        return a_nDefault;
+    }
+
+    // handle the value as hex if prefaced with "0x"
+    long nValue = a_nDefault;
+    char * pszSuffix = szValue;
+    if (szValue[0] == '0' && (szValue[1] == 'x' || szValue[1] == 'X')) {
+    	if (!szValue[2]) return a_nDefault;
+        nValue = strtol(&szValue[2], &pszSuffix, 16);
+    }
+    else {
+        nValue = strtol(szValue, &pszSuffix, 10);
+    }
+
+    // any invalid strings will return the default value
+    if (*pszSuffix) { 
+        return a_nDefault; 
+    }
+
+    return nValue;
+}
+
+template
+SI_Error 
+CSimpleIniTempl::SetLongValue(
+    const SI_CHAR * a_pSection,
+    const SI_CHAR * a_pKey,
+    long            a_nValue,
+    const SI_CHAR * a_pComment,
+    bool            a_bUseHex,
+    bool            a_bForceReplace
+    )
+{
+    // use SetValue to create sections
+    if (!a_pSection || !a_pKey) return SI_FAIL;
+
+    // convert to an ASCII string
+    char szInput[64];
+#if __STDC_WANT_SECURE_LIB__ && !_WIN32_WCE
+    sprintf_s(szInput, a_bUseHex ? "0x%lx" : "%ld", a_nValue);
+#else // !__STDC_WANT_SECURE_LIB__
+    sprintf(szInput, a_bUseHex ? "0x%lx" : "%ld", a_nValue);
+#endif // __STDC_WANT_SECURE_LIB__
+
+    // convert to output text
+    SI_CHAR szOutput[64];
+    SI_CONVERTER c(m_bStoreIsUtf8);
+    c.ConvertFromStore(szInput, strlen(szInput) + 1, 
+        szOutput, sizeof(szOutput) / sizeof(SI_CHAR));
+
+    // actually add it
+    return AddEntry(a_pSection, a_pKey, szOutput, a_pComment, a_bForceReplace, true);
+}
+
+template
+bool
+CSimpleIniTempl::GetBoolValue(
+    const SI_CHAR * a_pSection,
+    const SI_CHAR * a_pKey,
+    bool            a_bDefault,
+    bool *          a_pHasMultiple
+    ) const
+{
+    // return the default if we don't have a value
+    const SI_CHAR * pszValue = GetValue(a_pSection, a_pKey, NULL, a_pHasMultiple);
+    if (!pszValue || !*pszValue) return a_bDefault;
+
+    // we only look at the minimum number of characters
+    switch (pszValue[0]) {
+    case 't': case 'T': // true
+    case 'y': case 'Y': // yes
+    case '1':           // 1 (one)
+        return true;
+
+    case 'f': case 'F': // false
+    case 'n': case 'N': // no
+    case '0':           // 0 (zero)
+        return false;
+
+    case 'o': case 'O':
+        if (pszValue[1] == 'n' || pszValue[1] == 'N') return true;  // on
+        if (pszValue[1] == 'f' || pszValue[1] == 'F') return false; // off
+        break;
+    }
+
+    // no recognized value, return the default
+    return a_bDefault;
+}
+
+template
+SI_Error 
+CSimpleIniTempl::SetBoolValue(
+    const SI_CHAR * a_pSection,
+    const SI_CHAR * a_pKey,
+    bool            a_bValue,
+    const SI_CHAR * a_pComment,
+    bool            a_bForceReplace
+    )
+{
+    // use SetValue to create sections
+    if (!a_pSection || !a_pKey) return SI_FAIL;
+
+    // convert to an ASCII string
+    const char * pszInput = a_bValue ? "true" : "false";
+
+    // convert to output text
+    SI_CHAR szOutput[64];
+    SI_CONVERTER c(m_bStoreIsUtf8);
+    c.ConvertFromStore(pszInput, strlen(pszInput) + 1, 
+        szOutput, sizeof(szOutput) / sizeof(SI_CHAR));
+
+    // actually add it
+    return AddEntry(a_pSection, a_pKey, szOutput, a_pComment, a_bForceReplace, true);
+}
+    
+template
+bool
+CSimpleIniTempl::GetAllValues(
+    const SI_CHAR * a_pSection,
+    const SI_CHAR * a_pKey,
+    TNamesDepend &  a_values
+    ) const
+{
+    a_values.clear();
+
+    if (!a_pSection || !a_pKey) {
+        return false;
+    }
+    typename TSection::const_iterator iSection = m_data.find(a_pSection);
+    if (iSection == m_data.end()) {
+        return false;
+    }
+    typename TKeyVal::const_iterator iKeyVal = iSection->second.find(a_pKey);
+    if (iKeyVal == iSection->second.end()) {
+        return false;
+    }
+
+    // insert all values for this key
+    a_values.push_back(Entry(iKeyVal->second, iKeyVal->first.pComment, iKeyVal->first.nOrder));
+    if (m_bAllowMultiKey) {
+        ++iKeyVal;
+        while (iKeyVal != iSection->second.end() && !IsLess(a_pKey, iKeyVal->first.pItem)) {
+            a_values.push_back(Entry(iKeyVal->second, iKeyVal->first.pComment, iKeyVal->first.nOrder));
+            ++iKeyVal;
+        }
+    }
+
+    return true;
+}
+
+template
+int
+CSimpleIniTempl::GetSectionSize(
+    const SI_CHAR * a_pSection
+    ) const
+{
+    if (!a_pSection) {
+        return -1;
+    }
+
+    typename TSection::const_iterator iSection = m_data.find(a_pSection);
+    if (iSection == m_data.end()) {
+        return -1;
+    }
+    const TKeyVal & section = iSection->second;
+
+    // if multi-key isn't permitted then the section size is
+    // the number of keys that we have.
+    if (!m_bAllowMultiKey || section.empty()) {
+        return (int) section.size();
+    }
+
+    // otherwise we need to count them
+    int nCount = 0;
+    const SI_CHAR * pLastKey = NULL;
+    typename TKeyVal::const_iterator iKeyVal = section.begin();
+    for (int n = 0; iKeyVal != section.end(); ++iKeyVal, ++n) {
+        if (!pLastKey || IsLess(pLastKey, iKeyVal->first.pItem)) {
+            ++nCount;
+            pLastKey = iKeyVal->first.pItem;
+        }
+    }
+    return nCount;
+}
+
+template
+const typename CSimpleIniTempl::TKeyVal *
+CSimpleIniTempl::GetSection(
+    const SI_CHAR * a_pSection
+    ) const
+{
+    if (a_pSection) {
+        typename TSection::const_iterator i = m_data.find(a_pSection);
+        if (i != m_data.end()) {
+            return &(i->second);
+        }
+    }
+    return 0;
+}
+
+template
+void
+CSimpleIniTempl::GetAllSections(
+    TNamesDepend & a_names
+    ) const
+{
+    a_names.clear();
+    typename TSection::const_iterator i = m_data.begin();
+    for (int n = 0; i != m_data.end(); ++i, ++n ) {
+        a_names.push_back(i->first);
+    }
+}
+
+template
+bool
+CSimpleIniTempl::GetAllKeys(
+    const SI_CHAR * a_pSection,
+    TNamesDepend &  a_names
+    ) const
+{
+    a_names.clear();
+
+    if (!a_pSection) {
+        return false;
+    }
+
+    typename TSection::const_iterator iSection = m_data.find(a_pSection);
+    if (iSection == m_data.end()) {
+        return false;
+    }
+
+    const TKeyVal & section = iSection->second;
+    const SI_CHAR * pLastKey = NULL;
+    typename TKeyVal::const_iterator iKeyVal = section.begin();
+    for (int n = 0; iKeyVal != section.end(); ++iKeyVal, ++n ) {
+        if (!pLastKey || IsLess(pLastKey, iKeyVal->first.pItem)) {
+            a_names.push_back(iKeyVal->first);
+            pLastKey = iKeyVal->first.pItem;
+        }
+    }
+
+    return true;
+}
+
+template
+SI_Error
+CSimpleIniTempl::SaveFile(
+    const char *    a_pszFile,
+    bool            a_bAddSignature
+    ) const
+{
+    FILE * fp = NULL;
+#if __STDC_WANT_SECURE_LIB__ && !_WIN32_WCE
+    fopen_s(&fp, a_pszFile, "wb");
+#else // !__STDC_WANT_SECURE_LIB__
+    fp = fopen(a_pszFile, "wb");
+#endif // __STDC_WANT_SECURE_LIB__
+    if (!fp) return SI_FILE;
+    SI_Error rc = SaveFile(fp, a_bAddSignature);
+    fclose(fp);
+    return rc;
+}
+
+#ifdef SI_HAS_WIDE_FILE
+template
+SI_Error
+CSimpleIniTempl::SaveFile(
+    const SI_WCHAR_T *  a_pwszFile,
+    bool                a_bAddSignature
+    ) const
+{
+#ifdef _WIN32
+    FILE * fp = NULL;
+#if __STDC_WANT_SECURE_LIB__ && !_WIN32_WCE
+    _wfopen_s(&fp, a_pwszFile, L"wb");
+#else // !__STDC_WANT_SECURE_LIB__
+    fp = _wfopen(a_pwszFile, L"wb");
+#endif // __STDC_WANT_SECURE_LIB__
+    if (!fp) return SI_FILE;
+    SI_Error rc = SaveFile(fp, a_bAddSignature);
+    fclose(fp);
+    return rc;
+#else // !_WIN32 (therefore SI_CONVERT_ICU)
+    char szFile[256];
+    u_austrncpy(szFile, a_pwszFile, sizeof(szFile));
+    return SaveFile(szFile, a_bAddSignature);
+#endif // _WIN32
+}
+#endif // SI_HAS_WIDE_FILE
+
+template
+SI_Error
+CSimpleIniTempl::SaveFile(
+    FILE *  a_pFile,
+    bool    a_bAddSignature
+    ) const
+{
+    FileWriter writer(a_pFile);
+    return Save(writer, a_bAddSignature);
+}
+
+template
+SI_Error
+CSimpleIniTempl::Save(
+    OutputWriter &  a_oOutput,
+    bool            a_bAddSignature
+    ) const
+{
+    Converter convert(m_bStoreIsUtf8);
+
+    // add the UTF-8 signature if it is desired
+    if (m_bStoreIsUtf8 && a_bAddSignature) {
+        a_oOutput.Write(SI_UTF8_SIGNATURE);
+    }
+
+    // get all of the sections sorted in load order
+    TNamesDepend oSections;
+    GetAllSections(oSections);
+#if defined(_MSC_VER) && _MSC_VER <= 1200
+    oSections.sort();
+#elif defined(__BORLANDC__)
+    oSections.sort(Entry::LoadOrder());
+#else
+    oSections.sort(typename Entry::LoadOrder());
+#endif
+
+    // write the file comment if we have one
+    bool bNeedNewLine = false;
+    if (m_pFileComment) {
+        if (!OutputMultiLineText(a_oOutput, convert, m_pFileComment)) {
+            return SI_FAIL;
+        }
+        bNeedNewLine = true;
+    }
+
+    // iterate through our sections and output the data
+    typename TNamesDepend::const_iterator iSection = oSections.begin();
+    for ( ; iSection != oSections.end(); ++iSection ) {
+        // write out the comment if there is one
+        if (iSection->pComment) {
+            if (bNeedNewLine) {
+                a_oOutput.Write(SI_NEWLINE_A);
+                a_oOutput.Write(SI_NEWLINE_A);
+            }
+            if (!OutputMultiLineText(a_oOutput, convert, iSection->pComment)) {
+                return SI_FAIL;
+            }
+            bNeedNewLine = false;
+        }
+
+        if (bNeedNewLine) {
+            a_oOutput.Write(SI_NEWLINE_A);
+            a_oOutput.Write(SI_NEWLINE_A);
+            bNeedNewLine = false;
+        }
+
+        // write the section (unless there is no section name)
+        if (*iSection->pItem) {
+            if (!convert.ConvertToStore(iSection->pItem)) {
+                return SI_FAIL;
+            }
+            a_oOutput.Write("[");
+            a_oOutput.Write(convert.Data());
+            a_oOutput.Write("]");
+            a_oOutput.Write(SI_NEWLINE_A);
+        }
+
+        // get all of the keys sorted in load order
+        TNamesDepend oKeys;
+        GetAllKeys(iSection->pItem, oKeys);
+#if defined(_MSC_VER) && _MSC_VER <= 1200
+        oKeys.sort();
+#elif defined(__BORLANDC__)
+        oKeys.sort(Entry::LoadOrder());
+#else
+        oKeys.sort(typename Entry::LoadOrder());
+#endif
+
+        // write all keys and values
+        typename TNamesDepend::const_iterator iKey = oKeys.begin();
+        for ( ; iKey != oKeys.end(); ++iKey) {
+            // get all values for this key
+            TNamesDepend oValues;
+            GetAllValues(iSection->pItem, iKey->pItem, oValues);
+
+            typename TNamesDepend::const_iterator iValue = oValues.begin();
+            for ( ; iValue != oValues.end(); ++iValue) {
+                // write out the comment if there is one
+                if (iValue->pComment) {
+                    a_oOutput.Write(SI_NEWLINE_A);
+                    if (!OutputMultiLineText(a_oOutput, convert, iValue->pComment)) {
+                        return SI_FAIL;
+                    }
+                }
+
+                // write the key
+                if (!convert.ConvertToStore(iKey->pItem)) {
+                    return SI_FAIL;
+                }
+                a_oOutput.Write(convert.Data());
+
+                // write the value
+                if (!convert.ConvertToStore(iValue->pItem)) {
+                    return SI_FAIL;
+                }
+                a_oOutput.Write(m_bSpaces ? " = " : "=");
+                if (m_bAllowMultiLine && IsMultiLineData(iValue->pItem)) {
+                    // multi-line data needs to be processed specially to ensure
+                    // that we use the correct newline format for the current system
+                    a_oOutput.Write("<<pItem)) {
+                        return SI_FAIL;
+                    }
+                    a_oOutput.Write("SI-END-OF-MULTILINE-TEXT");
+                }
+                else {
+                    a_oOutput.Write(convert.Data());
+                }
+                a_oOutput.Write(SI_NEWLINE_A);
+            }
+        }
+
+        bNeedNewLine = true;
+    }
+
+    return SI_OK;
+}
+
+template
+bool
+CSimpleIniTempl::OutputMultiLineText(
+    OutputWriter &  a_oOutput,
+    Converter &     a_oConverter,
+    const SI_CHAR * a_pText
+    ) const
+{
+    const SI_CHAR * pEndOfLine;
+    SI_CHAR cEndOfLineChar = *a_pText;
+    while (cEndOfLineChar) {
+        // find the end of this line
+        pEndOfLine = a_pText;
+        for (; *pEndOfLine && *pEndOfLine != '\n'; ++pEndOfLine) /*loop*/ ;
+        cEndOfLineChar = *pEndOfLine;
+
+        // temporarily null terminate, convert and output the line
+        *const_cast(pEndOfLine) = 0;
+        if (!a_oConverter.ConvertToStore(a_pText)) {
+            return false;
+        }
+        *const_cast(pEndOfLine) = cEndOfLineChar;
+        a_pText += (pEndOfLine - a_pText) + 1;
+        a_oOutput.Write(a_oConverter.Data());
+        a_oOutput.Write(SI_NEWLINE_A);
+    }
+    return true;
+}
+
+template
+bool
+CSimpleIniTempl::Delete(
+    const SI_CHAR * a_pSection,
+    const SI_CHAR * a_pKey,
+    bool            a_bRemoveEmpty
+    )
+{
+    if (!a_pSection) {
+        return false;
+    }
+
+    typename TSection::iterator iSection = m_data.find(a_pSection);
+    if (iSection == m_data.end()) {
+        return false;
+    }
+
+    // remove a single key if we have a keyname
+    if (a_pKey) {
+        typename TKeyVal::iterator iKeyVal = iSection->second.find(a_pKey);
+        if (iKeyVal == iSection->second.end()) {
+            return false;
+        }
+
+        // remove any copied strings and then the key
+        typename TKeyVal::iterator iDelete;
+        do {
+            iDelete = iKeyVal++;
+
+            DeleteString(iDelete->first.pItem);
+            DeleteString(iDelete->second);
+            iSection->second.erase(iDelete);
+        }
+        while (iKeyVal != iSection->second.end()
+            && !IsLess(a_pKey, iKeyVal->first.pItem));
+
+        // done now if the section is not empty or we are not pruning away
+        // the empty sections. Otherwise let it fall through into the section
+        // deletion code
+        if (!a_bRemoveEmpty || !iSection->second.empty()) {
+            return true;
+        }
+    }
+    else {
+        // delete all copied strings from this section. The actual
+        // entries will be removed when the section is removed.
+        typename TKeyVal::iterator iKeyVal = iSection->second.begin();
+        for ( ; iKeyVal != iSection->second.end(); ++iKeyVal) {
+            DeleteString(iKeyVal->first.pItem);
+            DeleteString(iKeyVal->second);
+        }
+    }
+
+    // delete the section itself
+    DeleteString(iSection->first.pItem);
+    m_data.erase(iSection);
+
+    return true;
+}
+
+template
+void
+CSimpleIniTempl::DeleteString(
+    const SI_CHAR * a_pString
+    )
+{
+    // strings may exist either inside the data block, or they will be
+    // individually allocated and stored in m_strings. We only physically
+    // delete those stored in m_strings.
+    if (a_pString < m_pData || a_pString >= m_pData + m_uDataLen) {
+        typename TNamesDepend::iterator i = m_strings.begin();
+        for (;i != m_strings.end(); ++i) {
+            if (a_pString == i->pItem) {
+                delete[] const_cast(i->pItem);
+                m_strings.erase(i);
+                break;
+            }
+        }
+    }
+}
+
+// ---------------------------------------------------------------------------
+//                              CONVERSION FUNCTIONS
+// ---------------------------------------------------------------------------
+
+// Defines the conversion classes for different libraries. Before including
+// SimpleIni.h, set the converter that you wish you use by defining one of the
+// following symbols.
+//
+//  SI_CONVERT_GENERIC      Use the Unicode reference conversion library in
+//                          the accompanying files ConvertUTF.h/c
+//  SI_CONVERT_ICU          Use the IBM ICU conversion library. Requires
+//                          ICU headers on include path and icuuc.lib
+//  SI_CONVERT_WIN32        Use the Win32 API functions for conversion.
+
+#if !defined(SI_CONVERT_GENERIC) && !defined(SI_CONVERT_WIN32) && !defined(SI_CONVERT_ICU)
+# ifdef _WIN32
+#  define SI_CONVERT_WIN32
+# else
+#  define SI_CONVERT_GENERIC
+# endif
+#endif
+
+/**
+ * Generic case-sensitive less than comparison. This class returns numerically
+ * ordered ASCII case-sensitive text for all possible sizes and types of
+ * SI_CHAR.
+ */
+template
+struct SI_GenericCase {
+    bool operator()(const SI_CHAR * pLeft, const SI_CHAR * pRight) const {
+        long cmp;
+        for ( ;*pLeft && *pRight; ++pLeft, ++pRight) {
+            cmp = (long) *pLeft - (long) *pRight;
+            if (cmp != 0) {
+                return cmp < 0;
+            }
+        }
+        return *pRight != 0;
+    }
+};
+
+/**
+ * Generic ASCII case-insensitive less than comparison. This class returns
+ * numerically ordered ASCII case-insensitive text for all possible sizes
+ * and types of SI_CHAR. It is not safe for MBCS text comparison where
+ * ASCII A-Z characters are used in the encoding of multi-byte characters.
+ */
+template
+struct SI_GenericNoCase {
+    inline SI_CHAR locase(SI_CHAR ch) const {
+        return (ch < 'A' || ch > 'Z') ? ch : (ch - 'A' + 'a');
+    }
+    bool operator()(const SI_CHAR * pLeft, const SI_CHAR * pRight) const {
+        long cmp;
+        for ( ;*pLeft && *pRight; ++pLeft, ++pRight) {
+            cmp = (long) locase(*pLeft) - (long) locase(*pRight);
+            if (cmp != 0) {
+                return cmp < 0;
+            }
+        }
+        return *pRight != 0;
+    }
+};
+
+/**
+ * Null conversion class for MBCS/UTF-8 to char (or equivalent).
+ */
+template
+class SI_ConvertA {
+    bool m_bStoreIsUtf8;
+protected:
+    SI_ConvertA() { }
+public:
+    SI_ConvertA(bool a_bStoreIsUtf8) : m_bStoreIsUtf8(a_bStoreIsUtf8) { }
+
+    /* copy and assignment */
+    SI_ConvertA(const SI_ConvertA & rhs) { operator=(rhs); }
+    SI_ConvertA & operator=(const SI_ConvertA & rhs) {
+        m_bStoreIsUtf8 = rhs.m_bStoreIsUtf8;
+        return *this;
+    }
+
+    /** Calculate the number of SI_CHAR required for converting the input
+     * from the storage format. The storage format is always UTF-8 or MBCS.
+     *
+     * @param a_pInputData  Data in storage format to be converted to SI_CHAR.
+     * @param a_uInputDataLen Length of storage format data in bytes. This
+     *                      must be the actual length of the data, including
+     *                      NULL byte if NULL terminated string is required.
+     * @return              Number of SI_CHAR required by the string when
+     *                      converted. If there are embedded NULL bytes in the
+     *                      input data, only the string up and not including
+     *                      the NULL byte will be converted.
+     * @return              -1 cast to size_t on a conversion error.
+     */
+    size_t SizeFromStore(
+        const char *    a_pInputData,
+        size_t          a_uInputDataLen)
+    {
+        (void)a_pInputData;
+        SI_ASSERT(a_uInputDataLen != (size_t) -1);
+
+        // ASCII/MBCS/UTF-8 needs no conversion
+        return a_uInputDataLen;
+    }
+
+    /** Convert the input string from the storage format to SI_CHAR.
+     * The storage format is always UTF-8 or MBCS.
+     *
+     * @param a_pInputData  Data in storage format to be converted to SI_CHAR.
+     * @param a_uInputDataLen Length of storage format data in bytes. This
+     *                      must be the actual length of the data, including
+     *                      NULL byte if NULL terminated string is required.
+     * @param a_pOutputData Pointer to the output buffer to received the
+     *                      converted data.
+     * @param a_uOutputDataSize Size of the output buffer in SI_CHAR.
+     * @return              true if all of the input data was successfully
+     *                      converted.
+     */
+    bool ConvertFromStore(
+        const char *    a_pInputData,
+        size_t          a_uInputDataLen,
+        SI_CHAR *       a_pOutputData,
+        size_t          a_uOutputDataSize)
+    {
+        // ASCII/MBCS/UTF-8 needs no conversion
+        if (a_uInputDataLen > a_uOutputDataSize) {
+            return false;
+        }
+        memcpy(a_pOutputData, a_pInputData, a_uInputDataLen);
+        return true;
+    }
+
+    /** Calculate the number of char required by the storage format of this
+     * data. The storage format is always UTF-8 or MBCS.
+     *
+     * @param a_pInputData  NULL terminated string to calculate the number of
+     *                      bytes required to be converted to storage format.
+     * @return              Number of bytes required by the string when
+     *                      converted to storage format. This size always
+     *                      includes space for the terminating NULL character.
+     * @return              -1 cast to size_t on a conversion error.
+     */
+    size_t SizeToStore(
+        const SI_CHAR * a_pInputData)
+    {
+        // ASCII/MBCS/UTF-8 needs no conversion
+        return strlen((const char *)a_pInputData) + 1;
+    }
+
+    /** Convert the input string to the storage format of this data.
+     * The storage format is always UTF-8 or MBCS.
+     *
+     * @param a_pInputData  NULL terminated source string to convert. All of
+     *                      the data will be converted including the
+     *                      terminating NULL character.
+     * @param a_pOutputData Pointer to the buffer to receive the converted
+     *                      string.
+     * @param a_uOutputDataSize Size of the output buffer in char.
+     * @return              true if all of the input data, including the
+     *                      terminating NULL character was successfully
+     *                      converted.
+     */
+    bool ConvertToStore(
+        const SI_CHAR * a_pInputData,
+        char *          a_pOutputData,
+        size_t          a_uOutputDataSize)
+    {
+        // calc input string length (SI_CHAR type and size independent)
+        size_t uInputLen = strlen((const char *)a_pInputData) + 1;
+        if (uInputLen > a_uOutputDataSize) {
+            return false;
+        }
+
+        // ascii/UTF-8 needs no conversion
+        memcpy(a_pOutputData, a_pInputData, uInputLen);
+        return true;
+    }
+};
+
+
+// ---------------------------------------------------------------------------
+//                              SI_CONVERT_GENERIC
+// ---------------------------------------------------------------------------
+#ifdef SI_CONVERT_GENERIC
+
+#define SI_Case     SI_GenericCase
+#define SI_NoCase   SI_GenericNoCase
+
+#include 
+#include "ConvertUTF.h"
+
+/**
+ * Converts UTF-8 to a wchar_t (or equivalent) using the Unicode reference
+ * library functions. This can be used on all platforms.
+ */
+template
+class SI_ConvertW {
+    bool m_bStoreIsUtf8;
+protected:
+    SI_ConvertW() { }
+public:
+    SI_ConvertW(bool a_bStoreIsUtf8) : m_bStoreIsUtf8(a_bStoreIsUtf8) { }
+
+    /* copy and assignment */
+    SI_ConvertW(const SI_ConvertW & rhs) { operator=(rhs); }
+    SI_ConvertW & operator=(const SI_ConvertW & rhs) {
+        m_bStoreIsUtf8 = rhs.m_bStoreIsUtf8;
+        return *this;
+    }
+
+    /** Calculate the number of SI_CHAR required for converting the input
+     * from the storage format. The storage format is always UTF-8 or MBCS.
+     *
+     * @param a_pInputData  Data in storage format to be converted to SI_CHAR.
+     * @param a_uInputDataLen Length of storage format data in bytes. This
+     *                      must be the actual length of the data, including
+     *                      NULL byte if NULL terminated string is required.
+     * @return              Number of SI_CHAR required by the string when
+     *                      converted. If there are embedded NULL bytes in the
+     *                      input data, only the string up and not including
+     *                      the NULL byte will be converted.
+     * @return              -1 cast to size_t on a conversion error.
+     */
+    size_t SizeFromStore(
+        const char *    a_pInputData,
+        size_t          a_uInputDataLen)
+    {
+        SI_ASSERT(a_uInputDataLen != (size_t) -1);
+
+        if (m_bStoreIsUtf8) {
+            // worst case scenario for UTF-8 to wchar_t is 1 char -> 1 wchar_t
+            // so we just return the same number of characters required as for
+            // the source text.
+            return a_uInputDataLen;
+        }
+        else {
+            return mbstowcs(NULL, a_pInputData, a_uInputDataLen);
+        }
+    }
+
+    /** Convert the input string from the storage format to SI_CHAR.
+     * The storage format is always UTF-8 or MBCS.
+     *
+     * @param a_pInputData  Data in storage format to be converted to SI_CHAR.
+     * @param a_uInputDataLen Length of storage format data in bytes. This
+     *                       must be the actual length of the data, including
+     *                       NULL byte if NULL terminated string is required.
+     * @param a_pOutputData Pointer to the output buffer to received the
+     *                       converted data.
+     * @param a_uOutputDataSize Size of the output buffer in SI_CHAR.
+     * @return              true if all of the input data was successfully
+     *                       converted.
+     */
+    bool ConvertFromStore(
+        const char *    a_pInputData,
+        size_t          a_uInputDataLen,
+        SI_CHAR *       a_pOutputData,
+        size_t          a_uOutputDataSize)
+    {
+        if (m_bStoreIsUtf8) {
+            // This uses the Unicode reference implementation to do the
+            // conversion from UTF-8 to wchar_t. The required files are
+            // ConvertUTF.h and ConvertUTF.c which should be included in
+            // the distribution but are publically available from unicode.org
+            // at http://www.unicode.org/Public/PROGRAMS/CVTUTF/
+            ConversionResult retval;
+            const UTF8 * pUtf8 = (const UTF8 *) a_pInputData;
+            if (sizeof(wchar_t) == sizeof(UTF32)) {
+                UTF32 * pUtf32 = (UTF32 *) a_pOutputData;
+                retval = ConvertUTF8toUTF32(
+                    &pUtf8, pUtf8 + a_uInputDataLen,
+                    &pUtf32, pUtf32 + a_uOutputDataSize,
+                    lenientConversion);
+            }
+            else if (sizeof(wchar_t) == sizeof(UTF16)) {
+                UTF16 * pUtf16 = (UTF16 *) a_pOutputData;
+                retval = ConvertUTF8toUTF16(
+                    &pUtf8, pUtf8 + a_uInputDataLen,
+                    &pUtf16, pUtf16 + a_uOutputDataSize,
+                    lenientConversion);
+            }
+            return retval == conversionOK;
+        }
+        else {
+            size_t retval = mbstowcs(a_pOutputData,
+                a_pInputData, a_uOutputDataSize);
+            return retval != (size_t)(-1);
+        }
+    }
+
+    /** Calculate the number of char required by the storage format of this
+     * data. The storage format is always UTF-8 or MBCS.
+     *
+     * @param a_pInputData  NULL terminated string to calculate the number of
+     *                       bytes required to be converted to storage format.
+     * @return              Number of bytes required by the string when
+     *                       converted to storage format. This size always
+     *                       includes space for the terminating NULL character.
+     * @return              -1 cast to size_t on a conversion error.
+     */
+    size_t SizeToStore(
+        const SI_CHAR * a_pInputData)
+    {
+        if (m_bStoreIsUtf8) {
+            // worst case scenario for wchar_t to UTF-8 is 1 wchar_t -> 6 char
+            size_t uLen = 0;
+            while (a_pInputData[uLen]) {
+                ++uLen;
+            }
+            return (6 * uLen) + 1;
+        }
+        else {
+            size_t uLen = wcstombs(NULL, a_pInputData, 0);
+            if (uLen == (size_t)(-1)) {
+                return uLen;
+            }
+            return uLen + 1; // include NULL terminator
+        }
+    }
+
+    /** Convert the input string to the storage format of this data.
+     * The storage format is always UTF-8 or MBCS.
+     *
+     * @param a_pInputData  NULL terminated source string to convert. All of
+     *                       the data will be converted including the
+     *                       terminating NULL character.
+     * @param a_pOutputData Pointer to the buffer to receive the converted
+     *                       string.
+     * @param a_uOutputDataSize Size of the output buffer in char.
+     * @return              true if all of the input data, including the
+     *                       terminating NULL character was successfully
+     *                       converted.
+     */
+    bool ConvertToStore(
+        const SI_CHAR * a_pInputData,
+        char *          a_pOutputData,
+        size_t          a_uOutputDataSize
+        )
+    {
+        if (m_bStoreIsUtf8) {
+            // calc input string length (SI_CHAR type and size independent)
+            size_t uInputLen = 0;
+            while (a_pInputData[uInputLen]) {
+                ++uInputLen;
+            }
+            ++uInputLen; // include the NULL char
+
+            // This uses the Unicode reference implementation to do the
+            // conversion from wchar_t to UTF-8. The required files are
+            // ConvertUTF.h and ConvertUTF.c which should be included in
+            // the distribution but are publically available from unicode.org
+            // at http://www.unicode.org/Public/PROGRAMS/CVTUTF/
+            ConversionResult retval;
+            UTF8 * pUtf8 = (UTF8 *) a_pOutputData;
+            if (sizeof(wchar_t) == sizeof(UTF32)) {
+                const UTF32 * pUtf32 = (const UTF32 *) a_pInputData;
+                retval = ConvertUTF32toUTF8(
+                    &pUtf32, pUtf32 + uInputLen,
+                    &pUtf8, pUtf8 + a_uOutputDataSize,
+                    lenientConversion);
+            }
+            else if (sizeof(wchar_t) == sizeof(UTF16)) {
+                const UTF16 * pUtf16 = (const UTF16 *) a_pInputData;
+                retval = ConvertUTF16toUTF8(
+                    &pUtf16, pUtf16 + uInputLen,
+                    &pUtf8, pUtf8 + a_uOutputDataSize,
+                    lenientConversion);
+            }
+            return retval == conversionOK;
+        }
+        else {
+            size_t retval = wcstombs(a_pOutputData,
+                a_pInputData, a_uOutputDataSize);
+            return retval != (size_t) -1;
+        }
+    }
+};
+
+#endif // SI_CONVERT_GENERIC
+
+
+// ---------------------------------------------------------------------------
+//                              SI_CONVERT_ICU
+// ---------------------------------------------------------------------------
+#ifdef SI_CONVERT_ICU
+
+#define SI_Case     SI_GenericCase
+#define SI_NoCase   SI_GenericNoCase
+
+#include 
+
+/**
+ * Converts MBCS/UTF-8 to UChar using ICU. This can be used on all platforms.
+ */
+template
+class SI_ConvertW {
+    const char * m_pEncoding;
+    UConverter * m_pConverter;
+protected:
+    SI_ConvertW() : m_pEncoding(NULL), m_pConverter(NULL) { }
+public:
+    SI_ConvertW(bool a_bStoreIsUtf8) : m_pConverter(NULL) {
+        m_pEncoding = a_bStoreIsUtf8 ? "UTF-8" : NULL;
+    }
+
+    /* copy and assignment */
+    SI_ConvertW(const SI_ConvertW & rhs) { operator=(rhs); }
+    SI_ConvertW & operator=(const SI_ConvertW & rhs) {
+        m_pEncoding = rhs.m_pEncoding;
+        m_pConverter = NULL;
+        return *this;
+    }
+    ~SI_ConvertW() { if (m_pConverter) ucnv_close(m_pConverter); }
+
+    /** Calculate the number of UChar required for converting the input
+     * from the storage format. The storage format is always UTF-8 or MBCS.
+     *
+     * @param a_pInputData  Data in storage format to be converted to UChar.
+     * @param a_uInputDataLen Length of storage format data in bytes. This
+     *                      must be the actual length of the data, including
+     *                      NULL byte if NULL terminated string is required.
+     * @return              Number of UChar required by the string when
+     *                      converted. If there are embedded NULL bytes in the
+     *                      input data, only the string up and not including
+     *                      the NULL byte will be converted.
+     * @return              -1 cast to size_t on a conversion error.
+     */
+    size_t SizeFromStore(
+        const char *    a_pInputData,
+        size_t          a_uInputDataLen)
+    {
+        SI_ASSERT(a_uInputDataLen != (size_t) -1);
+
+        UErrorCode nError;
+
+        if (!m_pConverter) {
+            nError = U_ZERO_ERROR;
+            m_pConverter = ucnv_open(m_pEncoding, &nError);
+            if (U_FAILURE(nError)) {
+                return (size_t) -1;
+            }
+        }
+
+        nError = U_ZERO_ERROR;
+        ucnv_resetToUnicode(m_pConverter);
+        int32_t nLen = ucnv_toUChars(m_pConverter, NULL, 0,
+            a_pInputData, (int32_t) a_uInputDataLen, &nError);
+        if (nError != U_BUFFER_OVERFLOW_ERROR) {
+            return (size_t) -1;
+        }
+
+        return (size_t) nLen;
+    }
+
+    /** Convert the input string from the storage format to UChar.
+     * The storage format is always UTF-8 or MBCS.
+     *
+     * @param a_pInputData  Data in storage format to be converted to UChar.
+     * @param a_uInputDataLen Length of storage format data in bytes. This
+     *                      must be the actual length of the data, including
+     *                      NULL byte if NULL terminated string is required.
+     * @param a_pOutputData Pointer to the output buffer to received the
+     *                      converted data.
+     * @param a_uOutputDataSize Size of the output buffer in UChar.
+     * @return              true if all of the input data was successfully
+     *                      converted.
+     */
+    bool ConvertFromStore(
+        const char *    a_pInputData,
+        size_t          a_uInputDataLen,
+        UChar *         a_pOutputData,
+        size_t          a_uOutputDataSize)
+    {
+        UErrorCode nError;
+
+        if (!m_pConverter) {
+            nError = U_ZERO_ERROR;
+            m_pConverter = ucnv_open(m_pEncoding, &nError);
+            if (U_FAILURE(nError)) {
+                return false;
+            }
+        }
+
+        nError = U_ZERO_ERROR;
+        ucnv_resetToUnicode(m_pConverter);
+        ucnv_toUChars(m_pConverter,
+            a_pOutputData, (int32_t) a_uOutputDataSize,
+            a_pInputData, (int32_t) a_uInputDataLen, &nError);
+        if (U_FAILURE(nError)) {
+            return false;
+        }
+
+        return true;
+    }
+
+    /** Calculate the number of char required by the storage format of this
+     * data. The storage format is always UTF-8 or MBCS.
+     *
+     * @param a_pInputData  NULL terminated string to calculate the number of
+     *                      bytes required to be converted to storage format.
+     * @return              Number of bytes required by the string when
+     *                      converted to storage format. This size always
+     *                      includes space for the terminating NULL character.
+     * @return              -1 cast to size_t on a conversion error.
+     */
+    size_t SizeToStore(
+        const UChar * a_pInputData)
+    {
+        UErrorCode nError;
+
+        if (!m_pConverter) {
+            nError = U_ZERO_ERROR;
+            m_pConverter = ucnv_open(m_pEncoding, &nError);
+            if (U_FAILURE(nError)) {
+                return (size_t) -1;
+            }
+        }
+
+        nError = U_ZERO_ERROR;
+        ucnv_resetFromUnicode(m_pConverter);
+        int32_t nLen = ucnv_fromUChars(m_pConverter, NULL, 0,
+            a_pInputData, -1, &nError);
+        if (nError != U_BUFFER_OVERFLOW_ERROR) {
+            return (size_t) -1;
+        }
+
+        return (size_t) nLen + 1;
+    }
+
+    /** Convert the input string to the storage format of this data.
+     * The storage format is always UTF-8 or MBCS.
+     *
+     * @param a_pInputData  NULL terminated source string to convert. All of
+     *                      the data will be converted including the
+     *                      terminating NULL character.
+     * @param a_pOutputData Pointer to the buffer to receive the converted
+     *                      string.
+     * @param a_pOutputDataSize Size of the output buffer in char.
+     * @return              true if all of the input data, including the
+     *                      terminating NULL character was successfully
+     *                      converted.
+     */
+    bool ConvertToStore(
+        const UChar *   a_pInputData,
+        char *          a_pOutputData,
+        size_t          a_uOutputDataSize)
+    {
+        UErrorCode nError;
+
+        if (!m_pConverter) {
+            nError = U_ZERO_ERROR;
+            m_pConverter = ucnv_open(m_pEncoding, &nError);
+            if (U_FAILURE(nError)) {
+                return false;
+            }
+        }
+
+        nError = U_ZERO_ERROR;
+        ucnv_resetFromUnicode(m_pConverter);
+        ucnv_fromUChars(m_pConverter,
+            a_pOutputData, (int32_t) a_uOutputDataSize,
+            a_pInputData, -1, &nError);
+        if (U_FAILURE(nError)) {
+            return false;
+        }
+
+        return true;
+    }
+};
+
+#endif // SI_CONVERT_ICU
+
+
+// ---------------------------------------------------------------------------
+//                              SI_CONVERT_WIN32
+// ---------------------------------------------------------------------------
+#ifdef SI_CONVERT_WIN32
+
+#define SI_Case     SI_GenericCase
+
+// Windows CE doesn't have errno or MBCS libraries
+#ifdef _WIN32_WCE
+# ifndef SI_NO_MBCS
+#  define SI_NO_MBCS
+# endif
+#endif
+
+#include 
+#ifdef SI_NO_MBCS
+# define SI_NoCase   SI_GenericNoCase
+#else // !SI_NO_MBCS
+/**
+ * Case-insensitive comparison class using Win32 MBCS functions. This class
+ * returns a case-insensitive semi-collation order for MBCS text. It may not
+ * be safe for UTF-8 text returned in char format as we don't know what
+ * characters will be folded by the function! Therefore, if you are using
+ * SI_CHAR == char and SetUnicode(true), then you need to use the generic
+ * SI_NoCase class instead.
+ */
+#include 
+template
+struct SI_NoCase {
+    bool operator()(const SI_CHAR * pLeft, const SI_CHAR * pRight) const {
+        if (sizeof(SI_CHAR) == sizeof(char)) {
+            return _mbsicmp((const unsigned char *)pLeft,
+                (const unsigned char *)pRight) < 0;
+        }
+        if (sizeof(SI_CHAR) == sizeof(wchar_t)) {
+            return _wcsicmp((const wchar_t *)pLeft,
+                (const wchar_t *)pRight) < 0;
+        }
+        return SI_GenericNoCase()(pLeft, pRight);
+    }
+};
+#endif // SI_NO_MBCS
+
+/**
+ * Converts MBCS and UTF-8 to a wchar_t (or equivalent) on Windows. This uses
+ * only the Win32 functions and doesn't require the external Unicode UTF-8
+ * conversion library. It will not work on Windows 95 without using Microsoft
+ * Layer for Unicode in your application.
+ */
+template
+class SI_ConvertW {
+    UINT m_uCodePage;
+protected:
+    SI_ConvertW() { }
+public:
+    SI_ConvertW(bool a_bStoreIsUtf8) {
+        m_uCodePage = a_bStoreIsUtf8 ? CP_UTF8 : CP_ACP;
+    }
+
+    /* copy and assignment */
+    SI_ConvertW(const SI_ConvertW & rhs) { operator=(rhs); }
+    SI_ConvertW & operator=(const SI_ConvertW & rhs) {
+        m_uCodePage = rhs.m_uCodePage;
+        return *this;
+    }
+
+    /** Calculate the number of SI_CHAR required for converting the input
+     * from the storage format. The storage format is always UTF-8 or MBCS.
+     *
+     * @param a_pInputData  Data in storage format to be converted to SI_CHAR.
+     * @param a_uInputDataLen Length of storage format data in bytes. This
+     *                      must be the actual length of the data, including
+     *                      NULL byte if NULL terminated string is required.
+     * @return              Number of SI_CHAR required by the string when
+     *                      converted. If there are embedded NULL bytes in the
+     *                      input data, only the string up and not including
+     *                      the NULL byte will be converted.
+     * @return              -1 cast to size_t on a conversion error.
+     */
+    size_t SizeFromStore(
+        const char *    a_pInputData,
+        size_t          a_uInputDataLen)
+    {
+        SI_ASSERT(a_uInputDataLen != (size_t) -1);
+
+        int retval = MultiByteToWideChar(
+            m_uCodePage, 0,
+            a_pInputData, (int) a_uInputDataLen,
+            0, 0);
+        return (size_t)(retval > 0 ? retval : -1);
+    }
+
+    /** Convert the input string from the storage format to SI_CHAR.
+     * The storage format is always UTF-8 or MBCS.
+     *
+     * @param a_pInputData  Data in storage format to be converted to SI_CHAR.
+     * @param a_uInputDataLen Length of storage format data in bytes. This
+     *                      must be the actual length of the data, including
+     *                      NULL byte if NULL terminated string is required.
+     * @param a_pOutputData Pointer to the output buffer to received the
+     *                      converted data.
+     * @param a_uOutputDataSize Size of the output buffer in SI_CHAR.
+     * @return              true if all of the input data was successfully
+     *                      converted.
+     */
+    bool ConvertFromStore(
+        const char *    a_pInputData,
+        size_t          a_uInputDataLen,
+        SI_CHAR *       a_pOutputData,
+        size_t          a_uOutputDataSize)
+    {
+        int nSize = MultiByteToWideChar(
+            m_uCodePage, 0,
+            a_pInputData, (int) a_uInputDataLen,
+            (wchar_t *) a_pOutputData, (int) a_uOutputDataSize);
+        return (nSize > 0);
+    }
+
+    /** Calculate the number of char required by the storage format of this
+     * data. The storage format is always UTF-8.
+     *
+     * @param a_pInputData  NULL terminated string to calculate the number of
+     *                      bytes required to be converted to storage format.
+     * @return              Number of bytes required by the string when
+     *                      converted to storage format. This size always
+     *                      includes space for the terminating NULL character.
+     * @return              -1 cast to size_t on a conversion error.
+     */
+    size_t SizeToStore(
+        const SI_CHAR * a_pInputData)
+    {
+        int retval = WideCharToMultiByte(
+            m_uCodePage, 0,
+            (const wchar_t *) a_pInputData, -1,
+            0, 0, 0, 0);
+        return (size_t) (retval > 0 ? retval : -1);
+    }
+
+    /** Convert the input string to the storage format of this data.
+     * The storage format is always UTF-8 or MBCS.
+     *
+     * @param a_pInputData  NULL terminated source string to convert. All of
+     *                      the data will be converted including the
+     *                      terminating NULL character.
+     * @param a_pOutputData Pointer to the buffer to receive the converted
+     *                      string.
+     * @param a_pOutputDataSize Size of the output buffer in char.
+     * @return              true if all of the input data, including the
+     *                      terminating NULL character was successfully
+     *                      converted.
+     */
+    bool ConvertToStore(
+        const SI_CHAR * a_pInputData,
+        char *          a_pOutputData,
+        size_t          a_uOutputDataSize)
+    {
+        int retval = WideCharToMultiByte(
+            m_uCodePage, 0,
+            (const wchar_t *) a_pInputData, -1,
+            a_pOutputData, (int) a_uOutputDataSize, 0, 0);
+        return retval > 0;
+    }
+};
+
+#endif // SI_CONVERT_WIN32
+
+
+// ---------------------------------------------------------------------------
+//                                  TYPE DEFINITIONS
+// ---------------------------------------------------------------------------
+
+typedef CSimpleIniTempl,SI_ConvertA >                 CSimpleIniA;
+typedef CSimpleIniTempl,SI_ConvertA >                   CSimpleIniCaseA;
+
+#if defined(SI_CONVERT_ICU)
+typedef CSimpleIniTempl,SI_ConvertW >               CSimpleIniW;
+typedef CSimpleIniTempl,SI_ConvertW >                 CSimpleIniCaseW;
+#else
+typedef CSimpleIniTempl,SI_ConvertW >           CSimpleIniW;
+typedef CSimpleIniTempl,SI_ConvertW >             CSimpleIniCaseW;
+#endif
+
+#ifdef _UNICODE
+# define CSimpleIni      CSimpleIniW
+# define CSimpleIniCase  CSimpleIniCaseW
+# define SI_NEWLINE      SI_NEWLINE_W
+#else // !_UNICODE
+# define CSimpleIni      CSimpleIniA
+# define CSimpleIniCase  CSimpleIniCaseA
+# define SI_NEWLINE      SI_NEWLINE_A
+#endif // _UNICODE
+
+#ifdef _MSC_VER
+# pragma warning (pop)
+#endif
+
+#endif // INCLUDED_SimpleIni_h
+
diff -r 000000000000 -r 582cbe817f2c src/Support/StrobeList.h
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/src/Support/StrobeList.h	Fri Feb 12 12:31:23 2010 +0000
@@ -0,0 +1,143 @@
+// Copyright 2007-2010, Thomas Walters
+//
+// AIM-C: A C++ implementation of the Auditory Image Model
+// http://www.acousticscale.org/AIMC
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU General Public License for more details.
+//
+// You should have received a copy of the GNU General Public License
+// along with this program.  If not, see .
+
+/*!
+ * \file
+ * \brief Modifiable List of Strobe Points - helper for SAI generation
+ *
+ * \author Tom Walters 
+ * \date created 2007/08/22
+ * \version \$Id: StrobeList.h 1 2010-02-02 11:04:50Z tcw $
+ */
+
+#ifndef _AIMC_STROBE_LIST_H_
+#define _AIMC_STROBE_LIST_H_
+
+#include 
+
+#include "Support/util.h"
+
+typedef struct StrobePoint {
+  int iTime;
+  float fWeight;
+  float fWorkingWeight;
+  StrobePoint() {
+   iTime=0;
+   fWeight=0.0f;
+   fWorkingWeight=0.0f;
+  }
+};
+
+class StrobeList;
+/*!
+ * \class Signal "Support/StrobeList.h"
+ * \brief Modifiable List of Strobe Points, which must be ordered
+ *
+ */
+class StrobeList {
+ public:
+  /*! \brief Create a new strobe list
+   */
+  inline StrobeList() {
+    m_iStrobeCount = 0;
+    m_iMaxStrobes=0;
+    m_iOffset=0;
+    m_pStrobes = NULL;
+  };
+
+  inline void Create(unsigned int iMaxStrobes) {
+    m_iMaxStrobes = iMaxStrobes;
+    m_pStrobes = new StrobePoint[m_iMaxStrobes+1];
+  }
+
+  inline ~StrobeList() {
+    if (m_pStrobes != NULL)
+      delete [] m_pStrobes;
+  };
+
+  //! \brief Return the strobe time (in samples, can be negative)
+  inline unsigned int getTime(unsigned int iStrobeNumber) {
+    return m_pStrobes[GetBufferNumber(iStrobeNumber)].iTime;
+  };
+
+  //! \brief Return the strobe weight
+  inline float getWeight(unsigned int iStrobeNumber) {
+    return m_pStrobes[GetBufferNumber(iStrobeNumber)].fWeight;
+  };
+
+  //! \brief Return the strobe's working weight
+  inline float getWorkingWeight(unsigned int iStrobeNumber) {
+    return m_pStrobes[GetBufferNumber(iStrobeNumber)].fWorkingWeight;
+  };
+
+    //! \brief Set the strobe weight
+  inline void setWeight(unsigned int iStrobeNumber, float fWeight) {
+    m_pStrobes[GetBufferNumber(iStrobeNumber)].fWeight = fWeight;
+  };
+
+    //! \brief Set the strobe's working weight
+  inline void setWorkingWeight(unsigned int iStrobeNumber,
+                               float fWorkingWeight) {
+    m_pStrobes[GetBufferNumber(iStrobeNumber)].fWorkingWeight = fWorkingWeight;
+  };
+
+  //! \brief Add a strobe to the list (must be in order)
+  inline void addStrobe(int iTime, float fWeight) {
+    if (m_iStrobeCount + 1 <= m_iMaxStrobes) {
+      m_iStrobeCount++;
+      m_pStrobes[GetBufferNumber(m_iStrobeCount)].iTime=iTime;
+      m_pStrobes[GetBufferNumber(m_iStrobeCount)].fWeight=fWeight;
+    }
+  };
+
+  //! \brief Delete a strobe from the list
+  inline void deleteFirstStrobe() {
+    if (m_iStrobeCount > 0) {
+      m_iOffset = (m_iOffset+1) % m_iMaxStrobes;
+      m_iStrobeCount--;
+    }
+  };
+
+  //! \brief Get the number of strobes
+  inline unsigned int getStrobeCount() {
+    return m_iStrobeCount;
+  };
+
+  //! \brief Shift the position of all strobes by subtracting iOffset from
+  //! the time value of each
+  inline void shiftStrobes(unsigned int iOffset) {
+    for (unsigned int i=1; i.
+
+%module aimc 
+%{
+#include "Support/Common.h"
+#include "Support/Module.h"
+#include "Support/Parameters.h"
+#include "Support/SignalBank.h"
+#include "Modules/Features/ModuleGaussians.h"
+%}
+
+namespace aimc {
+class Parameters {
+public:
+	Parameters();
+	~Parameters();
+	bool Load(const char *sParamFilename);
+	bool Save(const char *sParamFilename);
+	bool Merge(const char *sParamFilename);
+  void SetDefault(const char* sName, const char* value);
+	void SetString(const char *sName, const char *val);
+	void SetInt(const char *sName, int val);
+	void SetUInt(const char *sName, unsigned int val);
+	void SetFloat(const char *sName, float val);
+	void SetBool(const char *sName, bool val);
+	const char *GetString(const char *sName);
+	int GetInt(const char *sName);
+	unsigned int GetUInt(const char *sName);
+	float GetFloat(const char *sName);
+	bool GetBool(const char *sName);
+	bool IsSet(const char *sName);
+	bool Parse(const char *sCmd);
+	bool Delete(const char *sName);
+	static const unsigned int MaxParamNameLength = 128;
+};
+
+class SignalBank {
+ public:
+  SignalBank();
+  bool Initialize(int channel_count, int signal_length, float sample_rate);
+  bool Validate() const;
+  inline const vector &operator[](int channel) const;
+  void set_sample(int channel, int sample, float value);
+  float sample_rate() const;
+  int buffer_length() const;
+  int start_time() const;
+  void set_start_time(int start_time);
+  float get_centre_frequency(int i) const;
+  void set_centre_frequency(int i, float cf);
+  bool initialized() const;
+  int channel_count() const;
+ private:
+  DISALLOW_COPY_AND_ASSIGN(SignalBank);
+};
+
+class Module {
+ public:
+  explicit Module(Parameters *parameters);
+  virtual ~Module();
+  bool Initialize(const SignalBank &input);
+  bool initialized() const;
+  bool AddTarget(Module* target_module);
+  bool DeleteTarget(Module* target_module);
+  void DeleteAllTargets();
+  virtual void Process(const SignalBank &input) = 0;
+  virtual void Reset() = 0;
+  const SignalBank* GetOutputBank() const;
+ private:
+  DISALLOW_COPY_AND_ASSIGN(Module);
+};
+
+class ModuleGaussians : public Module
+{
+ public:
+	ModuleGaussians(Parameters *pParam);
+	virtual ~ModuleGaussians();
+	virtual void Process(const SignalBank &input);
+	void Reset();
+};
+}  // namespace aimc
+
diff -r 000000000000 -r 582cbe817f2c swig/example.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/swig/example.py	Fri Feb 12 12:31:23 2010 +0000
@@ -0,0 +1,37 @@
+#!/usr/bin/env python
+# Copyright 2010, Thomas Walters
+#
+# AIM-C: A C++ implementation of the Auditory Image Model
+# http:#www.acousticscale.org/AIMC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+
+import aimc
+import matplotlib
+
+parameters = aimc.Parameters()
+parameters.LoadOnly('defaults.cfg')
+
+buffer_length = 1000
+sample_rate = 44100
+
+input_signal = aimc.SignalBank()
+input_signal.Initialize(1, buffer_length, sample_rate)
+
+filterbank = aimc.ModuleBMMPZFC(parameters)
+filterbank.Initialize(input_signal)
+
+filterbank_output = filterbank.getOutputBank()
+
+#for s in range(0, buffer_length):
diff -r 000000000000 -r 582cbe817f2c swig/setup.py
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/swig/setup.py	Fri Feb 12 12:31:23 2010 +0000
@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+# Copyright 2010, Thomas Walters
+#
+# AIM-C: A C++ implementation of the Auditory Image Model
+# http:#www.acousticscale.org/AIMC
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see .
+
+"""
+setup.py file for SWIG wrappers around aimc
+"""
+
+from distutils.core import setup, Extension
+
+aimc_module = Extension('_aimc',
+                        sources = ['aim_modules.i',
+                                   '../src/Support/Common.cc',
+                                   '../src/Support/Parameters.cc',
+                                   '../src/Support/Module.cc', 
+                                   '../src/Modules/Features/ModuleGaussians.cc'],
+                        swig_opts = ['-c++','-I../src/'], 
+                        include_dirs=['../src/']
+                        )
+
+setup (name = 'aimc',
+       version = '0.1',
+       author      = "Thomas Walters ",
+       description = """SWIG wrapper round the core of aimc""",
+       ext_modules = [aimc_module],
+       py_modules = ["aimc"],
+       )
\ No newline at end of file