Release freeDiameter v1.5.0main

6 files changed, 958 insertions, 20 deletions

diff --git a/contrib/tools/README b/contrib/tools/README
index bbb235d..11f5bad 100644
--- a/contrib/tools/README
+++ b/contrib/tools/README
@@ -1,2 +1,11 @@
+csv_to_fd converts CSV files containing RADIUS or Diameter AVP tables
+into various formats, including freeDiameter C code and JSON documents.
+
+grep_fd_dict_dump processes stdin for the output of fd_dict_dump()
+or dbg_dict_dump.fdx and reformats to remove pointer addresses,
+to allow diff of output between freeDiameter invocations.
+
+org_to_csv converts org files into CSV files, suitable for csv_to_fd.
+
 org_to_fd.pl converts org files like diameter-rfcs.org to C fragments
 that can be included in freeDiameter code.
diff --git a/contrib/tools/csv_to_fd b/contrib/tools/csv_to_fd
new file mode 100755
index 0000000..d39fc0a
--- /dev/null
+++ b/contrib/tools/csv_to_fd
@@ -0,0 +1,662 @@
+#!/usr/bin/env python
+
+"""
+Convert 8 column CSV files containing RADIUS or Diameter AVP tables
+into various formats.
+
+Format of the CSV files is one of:
+- Row per 3GPP AVP tables:
+    Name, Code, Section, DataType, Must, May, ShouldNot, MustNot
+    - Name:
+        AVP Name. String, validated as ALPHA *(ALPHA / DIGIT / "-")
+        per RFC 6733 section 3.2.
+        May start with a DIGIT (e.g., "3GPP-IMSI").
+    - Code:
+        AVP Code. Integer, 0..4294967295.
+    - Section:
+        Section in relevant standard. String.
+    - DataType:
+        AVP Data Type. String, validated per basic and derived types in:
+            - RFC 6733 section 4.2
+            - RFC 6733 section 4.3
+            - RFC 7155 section 4.1
+    - Must, May, ShouldNot, MustNot:
+        Flags, possibly comma or space separated: M, P, V
+
+- Comment row. First cell:
+    # Comment text      'Comment text'
+    #=                  '/*========*/'
+    #                   Blank line
+
+- Parameter row:
+    @Parameter,Value [, ...]
+  Supported Parameter terms:
+    standard    Standard name. E.g. '3GPP TS 29.272', 'RFC 6733'.
+    vendor      Vendor number.
+
+"""
+
+from __future__ import print_function
+from __future__ import with_statement
+
+import abc
+import csv
+import collections
+import json
+import re
+import optparse
+import os
+import sys
+
+CSV_COLUMN_NAMES = [
+    'name',
+    'code',
+    'section',
+    'datatype',
+    'must',
+    'may',
+    'shouldnot',
+    'mustnot',
+]
+
+DERIVED_TO_BASE = {
+    'Address':          'OctetString',  # RFC 6733 section 4.3.1
+    'Time':             'OctetString',  # RFC 6733 section 4.3.1
+    'UTF8String':       'OctetString',  # RFC 6733 section 4.3.1
+    'DiameterIdentity': 'OctetString',  # RFC 6733 section 4.3.1
+    'DiameterURI':      'OctetString',  # RFC 6733 section 4.3.1
+    'Enumerated':       'Integer32',    # RFC 6733 section 4.3.1
+    'IPFilterRule':     'OctetString',  # RFC 6733 section 4.3.1
+    'QoSFilterRule':    'OctetString',  # RFC 7155 section 4.1.1
+}
+
+# See https://www.iana.org/assignments/enterprise-numbers/enterprise-numbers
+VENDOR_TO_NAME = {
+    0:      '',
+    193:    'Ericsson',
+    8164:   'Starent',
+    10415:  '3GPP',
+}
+
+
+class Avp(object):
+    """Store an AVP row."""
+
+    # Regex to validate avp-name per RFC 6733 section 3.2,
+    # with changes:
+    # - Allow avp-name to start with numbers (for 3GPP)
+    # - Allow '.' in avp-name, for existing dict_dcca_3gpp usage.
+# TODO: if starts with digit, ensure contains a letter somewhere?
+    _name_re = re.compile(r'^[a-zA-Z0-9][a-zA-Z0-9-\.]*$')
+
+    # Regex to validate flags: M, P, V, comma, space
+    _flags_re = re.compile(r'^[MPV, ]*$')
+
+    __slots__ = CSV_COLUMN_NAMES + [
+        'filename', 'line_num', 'standard', 'vendor', ]
+
+    def __init__(self, name, code, section, datatype,
+                 must, may, shouldnot, mustnot, extra_cells=[],
+                 filename='', line_num=0, standard='', vendor=0):
+        # Members from CSV row
+        self.name = name
+        self.code = int(code)
+        self.section = section
+        self.datatype = datatype
+        self.must = must
+        self.may = may
+        self.shouldnot = shouldnot
+        self.mustnot = mustnot
+        # Members from file state
+        self.filename = filename
+        self.line_num = line_num
+        self.standard = standard
+        self.vendor = vendor
+        # Validate CSV fields
+        if not self._name_re.match(self.name):
+            raise ValueError('Invalid AVP name "{}"'.format(self.name))
+        if (self.code < 0 or self.code > 4294967295):
+            raise ValueError('AVP "{}" invalid code {}'.format(
+                self.name, self.code))
+        if (self.datatype not in (
+                'OctetString', 'Integer32', 'Integer64', 'Unsigned32',
+                'Unsigned64', 'Float32', 'Float64', 'Grouped')
+                and self.datatype not in DERIVED_TO_BASE):
+            raise ValueError('{} invalid data type "{}"'.format(
+                self.description(), self.datatype))
+        # Validate flags
+        flags = collections.Counter()
+        for val, desc in [
+                (self.must, 'Must'),
+                (self.may, 'May'),
+                (self.shouldnot, 'Should Not'),
+                (self.mustnot, 'Must Not'),
+                ]:
+            if not self._flags_re.match(val):
+                raise ValueError('{} invalid {} Flags "{}"'.format(
+                    self.description(), desc, val))
+            flags.update(val)
+        # Check occurrence of M,V in Must,May,ShouldNot,MustNot
+        for flag in 'MV':
+            # TODO: can AVP flags not appear at all?
+            # if flags[flag] == 0:
+            #     raise ValueError('{} Flag "{}" not set'.format(
+            #         self.description(), flag))
+            if flags[flag] > 1:
+                raise ValueError('{} Flag "{}" set {} times'.format(
+                    self.description(), flag, flags[flag]))
+        # Compare V presence against vendor
+        if 'V' in self.must:
+            if self.vendor == 0:
+                raise ValueError('{} Flag "V" set for vendor 0'.format(
+                    self.description()))
+        else:
+            if self.vendor != 0:
+                raise ValueError('{} Flag "V" not set for vendor {}'.format(
+                    self.description(), self.vendor))
+
+    @property
+    def __dict__(self):
+        return {s: getattr(self, s) for s in self.__slots__}
+
+    def __eq__(self, other):
+        """Equality comparison of Avp instances.
+        Considered equal if name, vendor, code, datatype, and flags are equal.
+        """
+        if other is self:
+            return True
+        if type(other) is not type(self):
+            return NotImplemented
+        return (
+            other.name, other.vendor, other.code, other.datatype,
+            other.must, other.may, other.shouldnot, other.mustnot,
+            ) == (
+            self.name, self.vendor, self.code, self.datatype,
+            self.must, self.may, self.shouldnot, self.mustnot,
+            )
+
+    def __ne__(self, other):
+        return not self == other
+
+    def description(self):
+        return 'AVP "{}" ({})'.format(self.name, self.code)
+
+
+class Processor(object):
+    """Interface for processor of Avp."""
+
+    __metaclass__ = abc.ABCMeta
+
+    @classmethod
+    def cls_name(cls):
+        """Return the name, lower-case, without "processor" suffix."""
+        suffix = 'processor'
+        name = cls.__name__.lower()
+        if name.endswith(suffix):
+            return name[:-len(suffix)]
+        return name
+
+    @classmethod
+    def cls_desc(cls):
+        """Return the first line of the docstring."""
+        if cls.__doc__ is None:
+            return ""
+        return cls.__doc__.split('\n')[0]
+
+    @abc.abstractmethod
+    def filename(self, filename):
+        """Called when a file is opened."""
+        pass
+
+    @abc.abstractmethod
+    def avp(self, avp):
+        """Process a validated Avp."""
+        pass
+
+    @abc.abstractmethod
+    def comment(self, comment, filename, line_num):
+        """Process a comment row:
+            #comment,
+        """
+        pass
+
+    @abc.abstractmethod
+    def generate(self):
+        """Invoked after all rows processed."""
+        pass
+
+    @abc.abstractmethod
+    def parameter(self, name, value):
+        """Process a parameter row:
+            @name,value.
+        """
+        pass
+
+
+class DebugProcessor(Processor):
+    """Display the CSV parsing."""
+
+    def filename(self, filename):
+        print('File: {}'.format(filename))
+
+    def avp(self, avp):
+        avpdict = vars(avp)
+        print('AVP: {name}, {code}, {datatype}'.format(**avpdict))
+
+    def comment(self, comment, filename, line_num):
+        print('Comment: {}'.format(comment))
+
+    def generate(self):
+        print('Generate')
+
+    def parameter(self, name, value):
+        print('Parameter: {} {}'.format(name, value))
+
+
+class NoopProcessor(Processor):
+    """Validate the CSV; no other output."""
+
+    def filename(self, filename):
+        pass
+
+    def avp(self, avp):
+        pass
+
+    def comment(self, comment, filename, line_num):
+        pass
+
+    def generate(self):
+        pass
+
+    def parameter(self, name, value):
+        pass
+
+
+class FdcProcessor(Processor):
+    """Generate freeDiameter C code.
+
+    Comment cells are parsed as:
+        # text comment  /* text comment */
+        #=              /*==============*/
+        #               [blank line]
+    """
+
+    COMMENT_WIDTH = 64
+
+    class AvpFunction(object):
+        """Maintain per-function state to create DICT_AVP entries.
+        """
+
+        def __init__(self, name):
+            self.__name = name
+            self.__lines = []
+            self.__derived = set()
+
+        @property
+        def name(self):
+            """Return name."""
+            return self.__name
+
+        @property
+        def lines(self):
+            """Return all lines."""
+            return self.__lines
+
+        @lines.setter
+        def lines(self, value):
+            """Set to append a line."""
+            self.__lines.append(value)
+
+        @property
+        def derived(self):
+            """Return list of all derived values."""
+            return list(self.__derived)
+
+        @derived.setter
+        def derived(self, value):
+            """Set to store a derived type."""
+            self.__derived.add(value)
+
+    def __init__(self):
+        self._filenames = []
+        self._functions = collections.OrderedDict()
+
+    def filename(self, filename):
+        self._filenames.append(os.path.basename(filename))
+
+    def avp(self, avp):
+        comment = '{name}, {datatype}, code {code}'.format(**vars(avp))
+        if avp.section != '':
+            comment += ', section {}'.format(avp.section)
+        self.add_comment(comment)
+        self.add('\t{')
+        self.add('\t\tstruct dict_avp_data data = {')
+# TODO: remove comments?
+        self.add('\t\t\t{},\t/* Code */'.format(avp.code))
+        self.add('\t\t\t{},\t/* Vendor */'.format(avp.vendor))
+        self.add('\t\t\t\"{}\",\t/* Name */'.format(avp.name))
+        self.add('\t\t\t{},\t/* Fixed flags */'.format(
+            self.build_flags(', '.join([avp.must, avp.mustnot]))))
+        self.add('\t\t\t{},\t/* Fixed flag values */'.format(
+            self.build_flags(avp.must)))
+# TODO: add trailing comma?
+        self.add('\t\t\tAVP_TYPE_{}\t/* base type of data */'.format(
+            DERIVED_TO_BASE.get(avp.datatype, avp.datatype).upper()))
+        self.add('\t\t};')
+        avp_type = 'NULL'
+        if avp.datatype == 'Enumerated':
+            self.add('\t\tstruct dict_object\t*type;')
+            vendor_prefix = ''
+            if avp.vendor != 0:
+                vendor_prefix = '{}/'.format(VENDOR_TO_NAME[avp.vendor])
+            self.add(
+                '\t\tstruct dict_type_data\t tdata = {{ AVP_TYPE_INTEGER32, '
+                '"Enumerated({prefix}{name})", NULL, NULL, NULL }};'.format(
+                    prefix=vendor_prefix, name=avp.name))
+# XXX: add enumerated values
+            self.add('\t\tCHECK_dict_new(DICT_TYPE, &tdata, NULL, &type);')
+            avp_type = "type"
+        elif avp.datatype in DERIVED_TO_BASE:
+            avp_type = '{}_type'.format(avp.datatype)
+            self.derived(avp.datatype)
+        self.add('\t\tCHECK_dict_new(DICT_AVP, &data, {}, NULL);'.format(
+            avp_type))
+# TODO: remove ; on scope brace
+        self.add('\t};')
+        self.add('')
+
+    def comment(self, comment, filename, line_num):
+        if comment == '':
+            self.add('')
+        elif comment == '=':
+            self.add_header()
+        elif comment.startswith(' '):
+            self.add_comment(comment[1:])
+        else:
+            raise ValueError('Unsupported comment "{}"'.format(comment))
+
+    def generate(self):
+        fp = sys.stdout
+        self.write_introduction(fp)
+        for func in self._functions:
+            self.write_function(fp, self._functions[func])
+
+    def parameter(self, name, value):
+        pass
+
+    # internal methods
+
+    def current_avpfunction(self):
+        """Return current AvpFunction to update.
+
+        Note: allows for easier future enhancement to generate separate
+        C functions per AVP groups such as: by csv file, standard, or vendor.
+        """
+        name = 'add_avps'
+        if name not in self._functions:
+            self._functions[name] = self.AvpFunction(name)
+        return self._functions[name]
+
+    def add(self, line):
+        self.current_avpfunction().lines = line
+
+    def derived(self, value):
+        self.current_avpfunction().derived = value
+
+    def build_c_token(self, value):
+        """Convert a string into a valid C token."""
+        return re.sub(r'[^\w]', '_', value)
+
+    def build_flags(self, flags):
+        result = []
+        if 'V' in flags:
+            result.append('AVP_FLAG_VENDOR')
+        if 'M' in flags:
+            result.append('AVP_FLAG_MANDATORY')
+        if not result:
+            return '0';
+        return ' |'.join(result)
+
+    def add_comment(self, comment):
+        self.add(self.format_comment(comment))
+
+    def add_header(self):
+        self.add(self.format_header())
+
+    def format_comment(self, comment):
+        return '\t/* {:<{width}} */'.format(comment, width=self.COMMENT_WIDTH)
+
+    def format_header(self):
+        return '\t/*={:=<{width}}=*/'.format('', width=self.COMMENT_WIDTH)
+
+    def write_introduction(self, fp):
+        """Write the introduction to the generated file."""
+        fp.write('''\
+/*
+Generated by:
+\tcsv_to_fd -p {processor} {files}
+
+Do not modify; modify the source .csv files instead.
+*/
+
+#include <freeDiameter/extension.h>
+
+#define CHECK_dict_new( _type, _data, _parent, _ref ) \\
+\tCHECK_FCT(  fd_dict_new( fd_g_config->cnf_dict, \
+(_type), (_data), (_parent), (_ref))  );
+
+#define CHECK_dict_search( _type, _criteria, _what, _result ) \\
+\tCHECK_FCT( fd_dict_search( fd_g_config->cnf_dict, \
+(_type), (_criteria), (_what), (_result), ENOENT) );
+'''.format(
+            processor=self.cls_name(),
+            files=' '.join(self._filenames)))
+
+    def write_function(self, fp, avpfunction):
+        """Generate a function from AvpFunction."""
+        function = self.build_c_token(avpfunction.name)
+        # Function start
+        fp.write('''\
+
+int {}()
+{{
+'''.format(function))
+
+        # Create variables used by derived type validation
+        for derived in avpfunction.derived:
+            fp.write('''\
+\tstruct dict_object * {name}_type = NULL;
+\tCHECK_dict_search(DICT_TYPE, TYPE_BY_NAME, "{name}", &{name}_type);
+
+'''.format(name=derived))
+
+        # Write generated DICT_AVP creation
+        fp.write('\n'.join(avpfunction.lines))
+
+        # Write function end
+        fp.write('''\
+
+\treturn 0;
+}} /* {}() */
+'''.format(function))
+
+
+class JsonProcessor(Processor):
+    """Generate freeDiameter JSON object.
+    """
+
+    def __init__(self):
+        self.avps = []
+
+    def filename(self, filename):
+        pass
+
+    def avp(self, avp):
+        flags = collections.OrderedDict([
+            ('Must',    self.build_flags(avp.must)),
+            ('MustNot', self.build_flags(avp.mustnot)),
+        ])
+        row = collections.OrderedDict([
+            ('Code',    avp.code),
+            ('Flags',   flags),
+            ('Name',    avp.name),
+            ('Type',    avp.datatype),
+            ('Vendor',  avp.vendor),
+        ])
+        self.avps.append(row)
+
+    def comment(self, comment, filename, line_num):
+        pass
+
+    def generate(self):
+        doc = {"AVPs": self.avps}
+        print(json.dumps(doc, indent=2))
+
+    def parameter(self, name, value):
+        pass
+
+    def build_flags(self, flags):
+        result = []
+        if 'V' in flags:
+            result.append('V')
+        if 'M' in flags:
+            result.append('M')
+        return ''.join(result)
+
+
+def avp_conflict(description, avp, conflict):
+    """Raise error for duplicate or conflicting AVPs.
+    """
+    if avp == conflict:
+        raise ValueError(
+            '{} {} duplicated in'
+            ' file "{}" line {}'.format(
+                avp.description(), description,
+                conflict.filename, conflict.line_num))
+    else:
+        raise ValueError(
+            '{} {} conflicts with {}'
+            ' in file "{}" line {}'.format(
+                avp.description(), description,
+                conflict.description(),
+                conflict.filename, conflict.line_num))
+
+
+def main():
+    """Main application entry.
+    """
+
+    # Build dict of name: NameProcessor
+    processors = {
+        cls.cls_name(): cls
+        for cls in Processor.__subclasses__()
+        }
+
+    # Build Processor name to desc
+    processor_help = '\n'.join(
+        ['  {:8} {}'.format(key, processors[key].cls_desc())
+         for key in sorted(processors)])
+
+    # Custom OptionParser with improved help
+    class MyParser(optparse.OptionParser):
+        """Custom OptionParser without epilog formatting."""
+        def format_help(self, formatter=None):
+            return """\
+{}
+Supported PROCESSOR options:
+{}
+""".format(
+                optparse.OptionParser.format_help(self, formatter),
+                processor_help)
+
+    # Parse options
+    parser = MyParser(
+        usage='%prog [-h] [-p PROCESSOR] FILE ...',
+        description="""\
+Convert CSV files FILE ... containing RADIUS or Diameter AVP tables
+into various formats using the specified processor PROCESSOR.
+""")
+    parser.add_option(
+        '-p', '--processor',
+        default='noop',
+        help='AVP processor. One of: {}. [%default]'.format(
+             ', '.join(processors.keys())))
+    (opts, args) = parser.parse_args()
+    if len(args) < 1:
+        parser.error('Incorrect number of arguments. Use -h for help.')
+
+    # Find processor
+    try:
+        avpproc = processors[opts.processor]()
+    except KeyError as e:
+        parser.error('Unknown processor "{}".'.format(opts.processor))
+
+    # dict of [vendor][code] : Avp
+    avp_codes = collections.defaultdict(dict)
+
+    # dict of [vendor][name] : Avp
+    avp_names = collections.defaultdict(dict)
+
+    # Process files
+    for filename in args:
+        avpproc.filename(filename)
+        with open(filename, 'r') as csvfile:
+            csvdata = csv.DictReader(csvfile, CSV_COLUMN_NAMES,
+                                     restkey='extra_cells', restval='')
+            standard = ''
+            vendor = 0
+            errors = []
+            for row in csvdata:
+                try:
+                    if csvdata.restkey in row:
+                        raise ValueError('Extra cells: {}'.format(
+                            ','.join(row[csvdata.restkey])))
+                    if row['name'] in (None, '', 'Attribute Name'):
+                        continue
+                    elif row['name'].startswith('#'):
+                        comment = row['name'][1:]
+                        avpproc.comment(comment, filename, csvdata.line_num)
+                    elif row['name'].startswith('@'):
+                        parameter = row['name'][1:]
+                        value = row['code']
+                        if False:
+                            pass
+                        elif parameter == 'standard':
+                            standard = value
+                        elif parameter == 'vendor':
+                            vendor = int(value)
+                        else:
+                            raise ValueError('Unknown parameter "{}"'.format(
+                                parameter))
+                        avpproc.parameter(parameter, value)
+                    else:
+                        avp = Avp(filename=filename, line_num=csvdata.line_num,
+                                  standard=standard, vendor=vendor,
+                                  **row)
+                        # Ensure AVP vendor/code not already defined
+                        if avp.code in avp_codes[avp.vendor]:
+                            conflict = avp_codes[avp.vendor][avp.code]
+                            avp_conflict('Code', avp, conflict)
+                        avp_codes[avp.vendor][avp.code] = avp
+                        # Ensure AVP vendor/name not already defined
+                        if avp.name in avp_names[avp.vendor]:
+                            conflict = avp_names[avp.vendor][avp.name]
+                            avp_conflict('Name', avp, conflict)
+                        avp_names[avp.vendor][avp.name] = avp
+                        # Process AVP
+                        avpproc.avp(avp)
+                except (TypeError, ValueError) as e:
+                    errors.append('CSV file "{}" line {}: {}\n'.format(
+                        filename, csvdata.line_num, e))
+            if errors:
+                sys.stderr.write(''.join(errors))
+                sys.exit(1)
+
+    # Generate result
+    avpproc.generate()
+
+
+if __name__ == '__main__':
+    main()
+
+# vim: set et sw=4 sts=4 :
diff --git a/contrib/tools/diameter-rfcs.csv b/contrib/tools/diameter-rfcs.csv
new file mode 100644
index 0000000..ed25e4b
--- /dev/null
+++ b/contrib/tools/diameter-rfcs.csv
@@ -0,0 +1,202 @@
+Attribute Name,Code,Section defined,Value Type,MUST,MAY,SHOULD NOT,MUST NOT

+@vendor,0,,,,,,

+@standard,RFC 3588,,,,,,

+#=,,,,,,,

+# RFC 3588 - Diameter #,,,,,,,

+#=,,,,,,,

+#,,,,,,,

+Acct-Interim-Interval,85,9.8.2,Unsigned32,M,P,,V

+Accounting-Realtime-Required,483,9.8.7,Enumerated,M,P,,V

+Acct-Multi-Session-Id,50,9.8.5,UTF8String,M,P,,V

+Accounting-Record-Number,485,9.8.3,Unsigned32,M,P,,V

+Accounting-Record-Type,480,9.8.1,Enumerated,M,P,,V

+Accounting-Session-Id,44,9.8.4,OctetString,M,P,,V

+Accounting-Sub-Session-Id,287,9.8.6,Unsigned64,M,P,,V

+Acct-Application-Id,259,6.9,Unsigned32,M,P,,V

+Auth-Application-Id,258,6.8,Unsigned32,M,P,,V

+Auth-Request-Type,274,8.7,Enumerated,M,P,,V

+Authorization-Lifetime,291,8.9,Unsigned32,M,P,,V

+Auth-Grace-Period,276,8.10,Unsigned32,M,P,,V

+Auth-Session-State,277,8.11,Enumerated,M,P,,V

+Re-Auth-Request-Type,285,8.12,Enumerated,M,P,,V

+Class,25,8.20,OctetString,M,P,,V

+Destination-Host,293,6.5,DiameterIdentity,M,P,,V

+Destination-Realm,283,6.6,DiameterIdentity,M,P,,V

+Disconnect-Cause,273,5.4.3,Enumerated,M,P,,V

+E2E-Sequence-AVP,300,6.15,Grouped,M,P,,V

+Error-Message,281,7.3,UTF8String,,P,,"V,M"

+Error-Reporting-Host,294,7.4,DiameterIdentity,,P,,"V,M"

+Event-Timestamp,55,8.21,Time,M,P,,V

+Experimental-Result,297,7.6,Grouped,M,P,,V

+Experimental-Result-Code,298,7.7,Unsigned32,M,P,,V

+Failed-AVP,279,7.5,Grouped,M,P,,V

+Firmware-Revision,267,5.3.4,Unsigned32,,,,"P,V,M"

+Host-IP-Address,257,5.3.5,Address,M,P,,V

+Inband-Security-Id,299,6.10,Unsigned32,M,P,,V

+Multi-Round-Time-Out,272,8.19,Unsigned32,M,P,,V

+Origin-Host,264,6.3,DiameterIdentity,M,P,,V

+Origin-Realm,296,6.4,DiameterIdentity,M,P,,V

+Origin-State-Id,278,8.16,Unsigned32,M,P,,V

+Product-Name,269,5.3.7,UTF8String,,,,"P,V,M"

+Proxy-Host,280,6.7.3,DiameterIdentity,M,,,"P,V"

+Proxy-Info,284,6.7.2,Grouped,M,,,"P,V"

+Proxy-State,33,6.7.4,OctetString,M,,,"P,V"

+Redirect-Host,292,6.12,DiameterURI,M,P,,V

+Redirect-Host-Usage,261,6.13,Enumerated,M,P,,V

+Redirect-Max-Cache-Time,262,6.14,Unsigned32,M,P,,V

+Result-Code,268,7.1,Unsigned32,M,P,,V

+Route-Record,282,6.7.1,DiameterIdentity,M,,,"P,V"

+Session-Id,263,8.8,UTF8String,M,P,,V

+Session-Timeout,27,8.13,Unsigned32,M,P,,V

+Session-Binding,270,8.17,Unsigned32,M,P,,V

+Session-Server-Failover,271,8.18,Enumerated,M,P,,V

+Supported-Vendor-Id,265,5.3.6,Unsigned32,M,P,,V

+Termination-Cause,295,8.15,Enumerated,M,P,,V

+User-Name,1,8.14,UTF8String,M,P,,V

+Vendor-Id,266,5.3.3,Unsigned32,M,P,,V

+Vendor-Specific-Application-Id,260,6.11,Grouped,M,P,,V

+@standard,RFC 4005,,,,,,

+#=,,,,,,,

+# RFC 4005 - NAS #,,,,,,,

+#=,,,,,,,

+#,,,,,,,

+NAS-Port,5,4.2,Unsigned32,M,P,,V

+NAS-Port-Id,87,4.3,UTF8String,M,P,,V

+NAS-Port-Type,61,4.4,Enumerated,M,P,,V

+Called-Station-Id,30,4.5,UTF8String,M,P,,V

+Calling-Station-Id,31,4.6,UTF8String,M,P,,V

+Connect-Info,77,4.7,UTF8String,M,P,,V

+Originating-Line-Info,94,4.8,OctetString,,"M,P",,V

+Reply-Message,18,4.9,UTF8String,M,P,,V

+User-Password,2,5.1,OctetString,M,P,,V

+Password-Retry,75,5.2,Unsigned32,M,P,,V

+Prompt,76,5.3,Enumerated,M,P,,V

+CHAP-Auth,402,5.4,Grouped,M,P,,V

+CHAP-Algorithm,403,5.5,Enumerated,M,P,,V

+CHAP-Ident,404,5.6,OctetString,M,P,,V

+CHAP-Response,405,5.7,OctetString,M,P,,V

+CHAP-Challenge,60,5.8,OctetString,M,P,,V

+ARAP-Password,70,5.9,OctetString,M,P,,V

+ARAP-Challenge-Response,84,5.10,OctetString,M,P,,V

+ARAP-Security,73,5.11,Unsigned32,M,P,,V

+ARAP-Security-Data,74,5.12,OctetString,M,P,,V

+Service-Type,6,6.1,Enumerated,M,P,,V

+Callback-Number,19,6.2,UTF8String,M,P,,V

+Callback-Id,20,6.3,UTF8String,M,P,,V

+Idle-Timeout,28,6.4,Unsigned32,M,P,,V

+Port-Limit,62,6.5,Unsigned32,M,P,,V

+NAS-Filter-Rule,400,6.6,IPFilterRule,M,P,,V

+Filter-Id,11,6.7,UTF8String,M,P,,V

+Configuration-Token,78,6.8,OctetString,M,,,"P,V"

+QoS-Filter-Rule,407,6.9,QoSFilterRule,,,,

+Framed-Protocol,7,6.10.1,Enumerated,M,P,,V

+Framed-Routing,10,6.10.2,Enumerated,M,P,,V

+Framed-MTU,12,6.10.3,Unsigned32,M,P,,V

+Framed-Compression,13,6.10.4,Enumerated,M,P,,V

+Framed-IP-Address,8,6.11.1,OctetString,M,P,,V

+Framed-IP-Netmask,9,6.11.2,OctetString,M,P,,V

+Framed-Route,22,6.11.3,UTF8String,M,P,,V

+Framed-Pool,88,6.11.4,OctetString,M,P,,V

+Framed-Interface-Id,96,6.11.5,Unsigned64,M,P,,V

+Framed-IPv6-Prefix,97,6.11.6,OctetString,M,P,,V

+Framed-IPv6-Route,99,6.11.7,UTF8String,M,P,,V

+Framed-IPv6-Pool,100,6.11.8,OctetString,M,P,,V

+Framed-IPX-Network,23,6.12.1,UTF8String,M,P,,V

+Framed-Appletalk-Link,37,6.13.1,Unsigned32,M,P,,V

+Framed-Appletalk-Network,38,6.13.2,Unsigned32,M,P,,V

+Framed-Appletalk-Zone,39,6.13.3,OctetString,M,P,,V

+ARAP-Features,71,6.14.1,OctetString,M,P,,V

+ARAP-Zone-Access,72,6.14.2,Enumerated,M,P,,V

+Login-IP-Host,14,6.15.1,OctetString,M,P,,V

+Login-IPv6-Host,98,6.15.2,OctetString,M,P,,V

+Login-Service,15,6.15.3,Enumerated,M,P,,V

+Login-TCP-Port,16,6.16.1,Unsigned32,M,P,,V

+Login-LAT-Service,34,6.17.1,OctetString,M,P,,V

+Login-LAT-Node,35,6.17.2,OctetString,M,P,,V

+Login-LAT-Group,36,6.17.3,OctetString,M,P,,V

+Login-LAT-Port,63,6.17.4,OctetString,M,P,,V

+Tunneling,401,7.1,Grouped,M,P,,V

+Tunnel-Type,64,7.2,Enumerated,M,P,,V

+Tunnel-Medium-Type,65,7.3,Enumerated,M,P,,V

+Tunnel-Client-Endpoint,66,7.4,UTF8String,M,P,,V

+Tunnel-Server-Endpoint,67,7.5,UTF8String,M,P,,V

+Tunnel-Password,69,7.6,OctetString,M,P,,V

+Tunnel-Private-Group-Id,81,7.7,OctetString,M,P,,V

+Tunnel-Assignment-Id,82,7.8,OctetString,M,P,,V

+Tunnel-Preference,83,7.9,Unsigned32,M,P,,V

+Tunnel-Client-Auth-Id,90,7.10,UTF8String,M,P,,V

+Tunnel-Server-Auth-Id,91,7.11,UTF8String,M,P,,V

+Accounting-Input-Octets,363,8.1,Unsigned64,M,P,,V

+Accounting-Output-Octets,364,8.2,Unsigned64,M,P,,V

+Accounting-Input-Packets,365,8.3,Unsigned64,M,P,,V

+Accounting-Output-Packets,366,8.4,Unsigned64,M,P,,V

+Acct-Session-Time,46,8.5,Unsigned32,M,P,,V

+Acct-Authentic,45,8.6,Enumerated,M,P,,V

+Acounting-Auth-Method,406,8.7,Enumerated,M,P,,V

+Acct-Delay-Time,41,8.8,Unsigned32,M,P,,V

+Acct-Link-Count,51,8.9,Unsigned32,M,P,,V

+Acct-Tunnel-Connection,68,8.10,OctetString,M,P,,V

+Acct-Tunnel-Packets-Lost,86,8.11,Unsigned32,M,P,,V

+NAS-Identifier,32,9.3.1,UTF8String,M,P,,V

+NAS-IP-Address,4,9.3.2,OctetString,M,P,,V

+NAS-IPv6-Address,95,9.3.3,OctetString,M,P,,V

+State,24,9.3.4,OctetString,M,P,,V

+# Termination-Cause already in RFC 3588,295,9.3.5,Enumerated,M,P,,V

+#,,,,,,,

+Origin-AAA-Protocol,408,9.3.6,Enumerated,M,P,,V

+@standard,RFC 4006,,,,,,

+#=,,,,,,,

+# RFC 4006 - DCCA #,,,,,,,

+#=,,,,,,,

+#,,,,,,,

+CC-Correlation-Id,411,8.1,OctetString,,"P,M",,V

+CC-Input-Octets,412,8.24,Unsigned64,M,P,,V

+CC-Money,413,8.22,Grouped,M,P,,V

+CC-Output-Octets,414,8.25,Unsigned64,M,P,,V

+CC-Request-Number,415,8.2,Unsigned32,M,P,,V

+CC-Request-Type,416,8.3,Enumerated,M,P,,V

+CC-Service-Specific-Units,417,8.26,Unsigned64,M,P,,V

+CC-Session-Failover,418,8.4,Enumerated,M,P,,V

+CC-Sub-Session-Id,419,8.5,Unsigned64,M,P,,V

+CC-Time,420,8.21,Unsigned32,M,P,,V

+CC-Total-Octets,421,8.23,Unsigned64,M,P,,V

+CC-Unit-Type,454,8.32,Enumerated,M,P,,V

+Check-Balance-Result,422,8.6,Enumerated,M,P,,V

+Cost-Information,423,8.7,Grouped,M,P,,V

+Cost-Unit,424,8.12,UTF8String,M,P,,V

+Credit-Control,426,8.13,Enumerated,M,P,,V

+Credit-Control-Failure-Handling,427,8.14,Enumerated,M,P,,V

+Currency-Code,425,8.11,Unsigned32,M,P,,V

+Direct-Debiting-Failure-Handling,428,8.15,Enumerated,M,P,,V

+Exponent,429,8.9,Integer32,M,P,,V

+Final-Unit-Action,449,8.35,Enumerated,M,P,,V

+Final-Unit-Indication,430,8.34,Grouped,M,P,,V

+Granted-Service-Unit,431,8.17,Grouped,M,P,,V

+G-S-U-Pool-Identifier,453,8.31,Unsigned32,M,P,,V

+G-S-U-Pool-Reference,457,8.30,Grouped,M,P,,V

+Multiple-Services-Credit-Control,456,8.16,Grouped,M,P,,V

+Multiple-Services-Indicator,455,8.40,Enumerated,M,P,,V

+Rating-Group,432,8.29,Unsigned32,M,P,,V

+Redirect-Address-Type,433,8.38,Enumerated,M,P,,V

+Redirect-Server,434,8.37,Grouped,M,P,,V

+Redirect-Server-Address,435,8.39,UTF8String,M,P,,V

+Requested-Action,436,8.41,Enumerated,M,P,,V

+Requested-Service-Unit,437,8.18,Grouped,M,P,,V

+Restriction-Filter-Rule,438,8.36,IPFilterRule,M,P,,V

+Service-Context-Id,461,8.42,UTF8String,M,P,,V

+Service-Identifier,439,8.28,Unsigned32,M,P,,V

+Service-Parameter-Info,440,8.43,Grouped,,"P,M",,V

+Service-Parameter-Type,441,8.44,Unsigned32,,"P,M",,V

+Service-Parameter-Value,442,8.45,OctetString,,"P,M",,V

+Subscription-Id,443,8.46,Grouped,M,P,,V

+Subscription-Id-Data,444,8.48,UTF8String,M,P,,V

+Subscription-Id-Type,450,8.47,Enumerated,M,P,,V

+Tariff-Change-Usage,452,8.27,Enumerated,M,P,,V

+Tariff-Time-Change,451,8.20,Time,M,P,,V

+Unit-Value,445,8.8,Grouped,M,P,,V

+Used-Service-Unit,446,8.19,Grouped,M,P,,V

+User-Equipment-Info,458,8.49,Grouped,,"P,M",,V

+User-Equipment-Info-Type,459,8.50,Enumerated,,"P,M",,V

+User-Equipment-Info-Value,460,8.51,OctetString,,"P,M",,V

+Value-Digits,447,8.10,Integer64,M,P,,V

+Validity-Time,448,8.33,Unsigned32,M,P,,V

diff --git a/contrib/tools/grep_fd_dict_dump b/contrib/tools/grep_fd_dict_dump
new file mode 100755
index 0000000..375fb5e
--- /dev/null
+++ b/contrib/tools/grep_fd_dict_dump
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+# Grep stdin for the output of fd_dict_dump() and reformat to remove
+# pointer addresses.  Use to post-process the output of dict_dump.fdx
+# into a format that's diff-able between freeDiameter invocations.
+
+egrep '^ *{dict|VENDOR|APPLICATION|TYPE|ENUMVAL|AVP|COMMAND|RULE' \
+	| sed \
+	-e 's/{dict.*}(@0x[^ ]*): //' \
+	-e 's/{dict(0x[^ ]*) : \(.*\)}/\1/' \
+	-e 's/p:[^ ]* //'
diff --git a/contrib/tools/org_to_csv b/contrib/tools/org_to_csv
new file mode 100755
index 0000000..e87f6ad
--- /dev/null
+++ b/contrib/tools/org_to_csv
@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+
+"""
+Convert |-separated 11-column .org files to CSV,
+with first and last empty columns ignored.
+"""
+
+import csv
+import fileinput
+import re
+import sys
+
+csvout = csv.writer(sys.stdout)
+for line in fileinput.input():
+    row = re.split(r'\s*\|\s*', line)
+    row.extend([''] * (10 - len(row)))
+    csvout.writerow(row[1:10])
+
+# vim: set et sw=4 sts=4 :
diff --git a/contrib/tools/org_to_fd.pl b/contrib/tools/org_to_fd.pl
index 96c06b7..a0825d6 100755
--- a/contrib/tools/org_to_fd.pl
+++ b/contrib/tools/org_to_fd.pl
@@ -1,12 +1,15 @@
 #!/usr/bin/env perl
 use strict;
+use File::Basename;
 use Getopt::Std;
 
+my ($progname) = basename($0);
+
 our ($opt_V, $opt_v);
 
-# default to 3GPP
-my ($vendor) = 10415;
-my ($vendor_name) = "3GPP";
+# default to Base
+my ($vendor) = 0;
+my ($vendor_name) = "";
 
 sub convert_must_to_flags($) {
     my ($allmust) = @_;
@@ -21,7 +24,7 @@ sub base_type($) {
     my ($type) = @_;
 
     return "AVP_TYPE_GROUPED" if ($type =~ m/Grouped/);
-    return "AVP_TYPE_OCTETSTRING" if ($type =~ m/(Address|DiameterIdentity|DiameterURI|OctetString|IPFilterRule|Time|UTF8String)/);
+    return "AVP_TYPE_OCTETSTRING" if ($type =~ m/(Address|DiameterIdentity|DiameterURI|OctetString|IPFilterRule|Time|UTF8String|QoSFilterRule)/);
     return "AVP_TYPE_INTEGER32" if ($type =~ m/Enumerated|Integer32/);
     return "AVP_TYPE_INTEGER64" if ($type =~ m/Integer64/);
     return "AVP_TYPE_UNSIGNED32" if ($type =~ m/Unsigned32/);
@@ -29,7 +32,19 @@ sub base_type($) {
     return "AVP_TYPE_FLOAT32" if ($type =~ m/Float32/);
     return "AVP_TYPE_FLOAT64" if ($type =~ m/Float64/);
 
-    return "UNKNOWN TYPE: $type";
+    die("unknown type '$type'");
+}
+
+
+my ($comment_width) = 64;
+
+sub print_header() {
+    printf "\t/*=%s=*/\n", '=' x $comment_width;
+}
+
+sub print_comment($) {
+    my ($str) = @_;
+    printf "\t/* %-*s */\n", $comment_width, $str;
 }
 
 sub print_insert($$) {
@@ -39,8 +54,8 @@ sub print_insert($$) {
     if ($type =~ m/(Grouped|OctetString|Integer32|Integer64|Unsigned32|Unsigned64|Float32|Float64)/) {
         $avp_type = "NULL";
     } elsif ($type =~ m/Enumerated/) {
-        print "\t\tstruct dict_object		*type;\n";
-        print "\t\tstruct dict_type_data	 tdata = { AVP_TYPE_INTEGER32, \"Enumerated(" . ($vendor_name ? "$vendor_name/" : "") ."$name)\", NULL, NULL, NULL };\n";
+        print "\t\tstruct dict_object\t*type;\n";
+        print "\t\tstruct dict_type_data\t tdata = { AVP_TYPE_INTEGER32, \"Enumerated(" . ($vendor_name ? "$vendor_name/" : "") ."$name)\", NULL, NULL, NULL };\n";
         # XXX: add enumerated values
         print "\t\tCHECK_dict_new(DICT_TYPE, &tdata, NULL, &type);\n";
         $avp_type = "type";
@@ -52,8 +67,8 @@ sub print_insert($$) {
 }
 
 sub usage($) {
-    die("usage: org_to_fd.pl [-V vendor_name -v vendor_code] [file ...]\n");
-    exit(@_);
+    print STDERR "usage: $progname [-V vendor_name] [-v vendor_code] [file ...]\n";
+    exit(1);
 }
 
 getopts("V:v:") || usage(1);
@@ -61,29 +76,45 @@ getopts("V:v:") || usage(1);
 if (defined($opt_v)) {
     $vendor = $opt_v;
     if (!defined($opt_V)) {
-	usage(1);
+        usage(1);
     }
     $vendor_name = $opt_V;
 }
 
-print "\t/* The following is created automatically. Do not modify. */\n";
-print "\t/* Changes will be lost during the next update. Modify the source org file instead. */\n\n";
+print_header();
+print_comment("Start of generated data.");
+print_comment("");
+print_comment("The following is created automatically with:");
+print_comment("    org_to_fd.pl -V '$vendor_name' -v $vendor");
+print_comment("Changes will be lost during the next update.");
+print_comment("Do not modify; modify the source .org file instead.");
+print_header();
+print "\n";
 
 while (<>) {
-    my ($dummy, $name, $code, $section, $type, $must, $may, $shouldnot, $mustnot, $encr) = split /\|/;
+    my ($dummy, $name, $code, $section, $type, $must, $may, $shouldnot, $mustnot, $encr) = split /\s*\|\s*/;
 
     next if ($name =~ m/Attribute Name/);
-    if ($name =~ m/ # (.*)/) {
-        print "\t/* $1 */\n";
+    if ($name =~ m/# (.*)/) {
+        print_comment($1);
+        next;
+    }
+    if ($name =~ m/#/) {
+        print("\n");
         next;
     }
-        
 
-    $name =~ s/ *//g;
-    $code =~ s/ *//g;
-    $type =~ s/ *//g;
+    if ($name =~ m/\s/) {
+        die("name '$name' contains space");
+    }
 
-    print "\t/* $name */\n\t{\n\t\tstruct dict_avp_data data = {\n";
+    my ($desc) = $name;
+    $desc .= ", " . $type;
+    $desc .= ", code " . $code;
+    $desc .= ", section " . $section if $section ne "";
+    print_comment($desc);
+    print "\t{\n";
+    print "\t\tstruct dict_avp_data data = {\n";
     print "\t\t\t$code,\t/* Code */\n";
     print "\t\t\t$vendor,\t/* Vendor */\n";
     print "\t\t\t\"$name\",\t/* Name */\n";
@@ -94,3 +125,7 @@ while (<>) {
     print_insert($type, $name);
     print "\t};\n\n";
 }
+
+print_header();
+print_comment("End of generated data.");
+print_header();