diff options
Diffstat (limited to 'contrib/tools/csv_to_fd')
| -rwxr-xr-x | contrib/tools/csv_to_fd | 662 |
1 files changed, 662 insertions, 0 deletions
diff --git a/contrib/tools/csv_to_fd b/contrib/tools/csv_to_fd new file mode 100755 index 0000000..d39fc0a --- /dev/null +++ b/contrib/tools/csv_to_fd @@ -0,0 +1,662 @@ +#!/usr/bin/env python + +""" +Convert 8 column CSV files containing RADIUS or Diameter AVP tables +into various formats. + +Format of the CSV files is one of: +- Row per 3GPP AVP tables: + Name, Code, Section, DataType, Must, May, ShouldNot, MustNot + - Name: + AVP Name. String, validated as ALPHA *(ALPHA / DIGIT / "-") + per RFC 6733 section 3.2. + May start with a DIGIT (e.g., "3GPP-IMSI"). + - Code: + AVP Code. Integer, 0..4294967295. + - Section: + Section in relevant standard. String. + - DataType: + AVP Data Type. String, validated per basic and derived types in: + - RFC 6733 section 4.2 + - RFC 6733 section 4.3 + - RFC 7155 section 4.1 + - Must, May, ShouldNot, MustNot: + Flags, possibly comma or space separated: M, P, V + +- Comment row. First cell: + # Comment text 'Comment text' + #= '/*========*/' + # Blank line + +- Parameter row: + @Parameter,Value [, ...] + Supported Parameter terms: + standard Standard name. E.g. '3GPP TS 29.272', 'RFC 6733'. + vendor Vendor number. + +""" + +from __future__ import print_function +from __future__ import with_statement + +import abc +import csv +import collections +import json +import re +import optparse +import os +import sys + +CSV_COLUMN_NAMES = [ + 'name', + 'code', + 'section', + 'datatype', + 'must', + 'may', + 'shouldnot', + 'mustnot', +] + +DERIVED_TO_BASE = { + 'Address': 'OctetString', # RFC 6733 section 4.3.1 + 'Time': 'OctetString', # RFC 6733 section 4.3.1 + 'UTF8String': 'OctetString', # RFC 6733 section 4.3.1 + 'DiameterIdentity': 'OctetString', # RFC 6733 section 4.3.1 + 'DiameterURI': 'OctetString', # RFC 6733 section 4.3.1 + 'Enumerated': 'Integer32', # RFC 6733 section 4.3.1 + 'IPFilterRule': 'OctetString', # RFC 6733 section 4.3.1 + 'QoSFilterRule': 'OctetString', # RFC 7155 section 4.1.1 +} + +# See https://www.iana.org/assignments/enterprise-numbers/enterprise-numbers +VENDOR_TO_NAME = { + 0: '', + 193: 'Ericsson', + 8164: 'Starent', + 10415: '3GPP', +} + + +class Avp(object): + """Store an AVP row.""" + + # Regex to validate avp-name per RFC 6733 section 3.2, + # with changes: + # - Allow avp-name to start with numbers (for 3GPP) + # - Allow '.' in avp-name, for existing dict_dcca_3gpp usage. +# TODO: if starts with digit, ensure contains a letter somewhere? + _name_re = re.compile(r'^[a-zA-Z0-9][a-zA-Z0-9-\.]*$') + + # Regex to validate flags: M, P, V, comma, space + _flags_re = re.compile(r'^[MPV, ]*$') + + __slots__ = CSV_COLUMN_NAMES + [ + 'filename', 'line_num', 'standard', 'vendor', ] + + def __init__(self, name, code, section, datatype, + must, may, shouldnot, mustnot, extra_cells=[], + filename='', line_num=0, standard='', vendor=0): + # Members from CSV row + self.name = name + self.code = int(code) + self.section = section + self.datatype = datatype + self.must = must + self.may = may + self.shouldnot = shouldnot + self.mustnot = mustnot + # Members from file state + self.filename = filename + self.line_num = line_num + self.standard = standard + self.vendor = vendor + # Validate CSV fields + if not self._name_re.match(self.name): + raise ValueError('Invalid AVP name "{}"'.format(self.name)) + if (self.code < 0 or self.code > 4294967295): + raise ValueError('AVP "{}" invalid code {}'.format( + self.name, self.code)) + if (self.datatype not in ( + 'OctetString', 'Integer32', 'Integer64', 'Unsigned32', + 'Unsigned64', 'Float32', 'Float64', 'Grouped') + and self.datatype not in DERIVED_TO_BASE): + raise ValueError('{} invalid data type "{}"'.format( + self.description(), self.datatype)) + # Validate flags + flags = collections.Counter() + for val, desc in [ + (self.must, 'Must'), + (self.may, 'May'), + (self.shouldnot, 'Should Not'), + (self.mustnot, 'Must Not'), + ]: + if not self._flags_re.match(val): + raise ValueError('{} invalid {} Flags "{}"'.format( + self.description(), desc, val)) + flags.update(val) + # Check occurrence of M,V in Must,May,ShouldNot,MustNot + for flag in 'MV': + # TODO: can AVP flags not appear at all? + # if flags[flag] == 0: + # raise ValueError('{} Flag "{}" not set'.format( + # self.description(), flag)) + if flags[flag] > 1: + raise ValueError('{} Flag "{}" set {} times'.format( + self.description(), flag, flags[flag])) + # Compare V presence against vendor + if 'V' in self.must: + if self.vendor == 0: + raise ValueError('{} Flag "V" set for vendor 0'.format( + self.description())) + else: + if self.vendor != 0: + raise ValueError('{} Flag "V" not set for vendor {}'.format( + self.description(), self.vendor)) + + @property + def __dict__(self): + return {s: getattr(self, s) for s in self.__slots__} + + def __eq__(self, other): + """Equality comparison of Avp instances. + Considered equal if name, vendor, code, datatype, and flags are equal. + """ + if other is self: + return True + if type(other) is not type(self): + return NotImplemented + return ( + other.name, other.vendor, other.code, other.datatype, + other.must, other.may, other.shouldnot, other.mustnot, + ) == ( + self.name, self.vendor, self.code, self.datatype, + self.must, self.may, self.shouldnot, self.mustnot, + ) + + def __ne__(self, other): + return not self == other + + def description(self): + return 'AVP "{}" ({})'.format(self.name, self.code) + + +class Processor(object): + """Interface for processor of Avp.""" + + __metaclass__ = abc.ABCMeta + + @classmethod + def cls_name(cls): + """Return the name, lower-case, without "processor" suffix.""" + suffix = 'processor' + name = cls.__name__.lower() + if name.endswith(suffix): + return name[:-len(suffix)] + return name + + @classmethod + def cls_desc(cls): + """Return the first line of the docstring.""" + if cls.__doc__ is None: + return "" + return cls.__doc__.split('\n')[0] + + @abc.abstractmethod + def filename(self, filename): + """Called when a file is opened.""" + pass + + @abc.abstractmethod + def avp(self, avp): + """Process a validated Avp.""" + pass + + @abc.abstractmethod + def comment(self, comment, filename, line_num): + """Process a comment row: + #comment, + """ + pass + + @abc.abstractmethod + def generate(self): + """Invoked after all rows processed.""" + pass + + @abc.abstractmethod + def parameter(self, name, value): + """Process a parameter row: + @name,value. + """ + pass + + +class DebugProcessor(Processor): + """Display the CSV parsing.""" + + def filename(self, filename): + print('File: {}'.format(filename)) + + def avp(self, avp): + avpdict = vars(avp) + print('AVP: {name}, {code}, {datatype}'.format(**avpdict)) + + def comment(self, comment, filename, line_num): + print('Comment: {}'.format(comment)) + + def generate(self): + print('Generate') + + def parameter(self, name, value): + print('Parameter: {} {}'.format(name, value)) + + +class NoopProcessor(Processor): + """Validate the CSV; no other output.""" + + def filename(self, filename): + pass + + def avp(self, avp): + pass + + def comment(self, comment, filename, line_num): + pass + + def generate(self): + pass + + def parameter(self, name, value): + pass + + +class FdcProcessor(Processor): + """Generate freeDiameter C code. + + Comment cells are parsed as: + # text comment /* text comment */ + #= /*==============*/ + # [blank line] + """ + + COMMENT_WIDTH = 64 + + class AvpFunction(object): + """Maintain per-function state to create DICT_AVP entries. + """ + + def __init__(self, name): + self.__name = name + self.__lines = [] + self.__derived = set() + + @property + def name(self): + """Return name.""" + return self.__name + + @property + def lines(self): + """Return all lines.""" + return self.__lines + + @lines.setter + def lines(self, value): + """Set to append a line.""" + self.__lines.append(value) + + @property + def derived(self): + """Return list of all derived values.""" + return list(self.__derived) + + @derived.setter + def derived(self, value): + """Set to store a derived type.""" + self.__derived.add(value) + + def __init__(self): + self._filenames = [] + self._functions = collections.OrderedDict() + + def filename(self, filename): + self._filenames.append(os.path.basename(filename)) + + def avp(self, avp): + comment = '{name}, {datatype}, code {code}'.format(**vars(avp)) + if avp.section != '': + comment += ', section {}'.format(avp.section) + self.add_comment(comment) + self.add('\t{') + self.add('\t\tstruct dict_avp_data data = {') +# TODO: remove comments? + self.add('\t\t\t{},\t/* Code */'.format(avp.code)) + self.add('\t\t\t{},\t/* Vendor */'.format(avp.vendor)) + self.add('\t\t\t\"{}\",\t/* Name */'.format(avp.name)) + self.add('\t\t\t{},\t/* Fixed flags */'.format( + self.build_flags(', '.join([avp.must, avp.mustnot])))) + self.add('\t\t\t{},\t/* Fixed flag values */'.format( + self.build_flags(avp.must))) +# TODO: add trailing comma? + self.add('\t\t\tAVP_TYPE_{}\t/* base type of data */'.format( + DERIVED_TO_BASE.get(avp.datatype, avp.datatype).upper())) + self.add('\t\t};') + avp_type = 'NULL' + if avp.datatype == 'Enumerated': + self.add('\t\tstruct dict_object\t*type;') + vendor_prefix = '' + if avp.vendor != 0: + vendor_prefix = '{}/'.format(VENDOR_TO_NAME[avp.vendor]) + self.add( + '\t\tstruct dict_type_data\t tdata = {{ AVP_TYPE_INTEGER32, ' + '"Enumerated({prefix}{name})", NULL, NULL, NULL }};'.format( + prefix=vendor_prefix, name=avp.name)) +# XXX: add enumerated values + self.add('\t\tCHECK_dict_new(DICT_TYPE, &tdata, NULL, &type);') + avp_type = "type" + elif avp.datatype in DERIVED_TO_BASE: + avp_type = '{}_type'.format(avp.datatype) + self.derived(avp.datatype) + self.add('\t\tCHECK_dict_new(DICT_AVP, &data, {}, NULL);'.format( + avp_type)) +# TODO: remove ; on scope brace + self.add('\t};') + self.add('') + + def comment(self, comment, filename, line_num): + if comment == '': + self.add('') + elif comment == '=': + self.add_header() + elif comment.startswith(' '): + self.add_comment(comment[1:]) + else: + raise ValueError('Unsupported comment "{}"'.format(comment)) + + def generate(self): + fp = sys.stdout + self.write_introduction(fp) + for func in self._functions: + self.write_function(fp, self._functions[func]) + + def parameter(self, name, value): + pass + + # internal methods + + def current_avpfunction(self): + """Return current AvpFunction to update. + + Note: allows for easier future enhancement to generate separate + C functions per AVP groups such as: by csv file, standard, or vendor. + """ + name = 'add_avps' + if name not in self._functions: + self._functions[name] = self.AvpFunction(name) + return self._functions[name] + + def add(self, line): + self.current_avpfunction().lines = line + + def derived(self, value): + self.current_avpfunction().derived = value + + def build_c_token(self, value): + """Convert a string into a valid C token.""" + return re.sub(r'[^\w]', '_', value) + + def build_flags(self, flags): + result = [] + if 'V' in flags: + result.append('AVP_FLAG_VENDOR') + if 'M' in flags: + result.append('AVP_FLAG_MANDATORY') + if not result: + return '0'; + return ' |'.join(result) + + def add_comment(self, comment): + self.add(self.format_comment(comment)) + + def add_header(self): + self.add(self.format_header()) + + def format_comment(self, comment): + return '\t/* {:<{width}} */'.format(comment, width=self.COMMENT_WIDTH) + + def format_header(self): + return '\t/*={:=<{width}}=*/'.format('', width=self.COMMENT_WIDTH) + + def write_introduction(self, fp): + """Write the introduction to the generated file.""" + fp.write('''\ +/* +Generated by: +\tcsv_to_fd -p {processor} {files} + +Do not modify; modify the source .csv files instead. +*/ + +#include <freeDiameter/extension.h> + +#define CHECK_dict_new( _type, _data, _parent, _ref ) \\ +\tCHECK_FCT( fd_dict_new( fd_g_config->cnf_dict, \ +(_type), (_data), (_parent), (_ref)) ); + +#define CHECK_dict_search( _type, _criteria, _what, _result ) \\ +\tCHECK_FCT( fd_dict_search( fd_g_config->cnf_dict, \ +(_type), (_criteria), (_what), (_result), ENOENT) ); +'''.format( + processor=self.cls_name(), + files=' '.join(self._filenames))) + + def write_function(self, fp, avpfunction): + """Generate a function from AvpFunction.""" + function = self.build_c_token(avpfunction.name) + # Function start + fp.write('''\ + +int {}() +{{ +'''.format(function)) + + # Create variables used by derived type validation + for derived in avpfunction.derived: + fp.write('''\ +\tstruct dict_object * {name}_type = NULL; +\tCHECK_dict_search(DICT_TYPE, TYPE_BY_NAME, "{name}", &{name}_type); + +'''.format(name=derived)) + + # Write generated DICT_AVP creation + fp.write('\n'.join(avpfunction.lines)) + + # Write function end + fp.write('''\ + +\treturn 0; +}} /* {}() */ +'''.format(function)) + + +class JsonProcessor(Processor): + """Generate freeDiameter JSON object. + """ + + def __init__(self): + self.avps = [] + + def filename(self, filename): + pass + + def avp(self, avp): + flags = collections.OrderedDict([ + ('Must', self.build_flags(avp.must)), + ('MustNot', self.build_flags(avp.mustnot)), + ]) + row = collections.OrderedDict([ + ('Code', avp.code), + ('Flags', flags), + ('Name', avp.name), + ('Type', avp.datatype), + ('Vendor', avp.vendor), + ]) + self.avps.append(row) + + def comment(self, comment, filename, line_num): + pass + + def generate(self): + doc = {"AVPs": self.avps} + print(json.dumps(doc, indent=2)) + + def parameter(self, name, value): + pass + + def build_flags(self, flags): + result = [] + if 'V' in flags: + result.append('V') + if 'M' in flags: + result.append('M') + return ''.join(result) + + +def avp_conflict(description, avp, conflict): + """Raise error for duplicate or conflicting AVPs. + """ + if avp == conflict: + raise ValueError( + '{} {} duplicated in' + ' file "{}" line {}'.format( + avp.description(), description, + conflict.filename, conflict.line_num)) + else: + raise ValueError( + '{} {} conflicts with {}' + ' in file "{}" line {}'.format( + avp.description(), description, + conflict.description(), + conflict.filename, conflict.line_num)) + + +def main(): + """Main application entry. + """ + + # Build dict of name: NameProcessor + processors = { + cls.cls_name(): cls + for cls in Processor.__subclasses__() + } + + # Build Processor name to desc + processor_help = '\n'.join( + [' {:8} {}'.format(key, processors[key].cls_desc()) + for key in sorted(processors)]) + + # Custom OptionParser with improved help + class MyParser(optparse.OptionParser): + """Custom OptionParser without epilog formatting.""" + def format_help(self, formatter=None): + return """\ +{} +Supported PROCESSOR options: +{} +""".format( + optparse.OptionParser.format_help(self, formatter), + processor_help) + + # Parse options + parser = MyParser( + usage='%prog [-h] [-p PROCESSOR] FILE ...', + description="""\ +Convert CSV files FILE ... containing RADIUS or Diameter AVP tables +into various formats using the specified processor PROCESSOR. +""") + parser.add_option( + '-p', '--processor', + default='noop', + help='AVP processor. One of: {}. [%default]'.format( + ', '.join(processors.keys()))) + (opts, args) = parser.parse_args() + if len(args) < 1: + parser.error('Incorrect number of arguments. Use -h for help.') + + # Find processor + try: + avpproc = processors[opts.processor]() + except KeyError as e: + parser.error('Unknown processor "{}".'.format(opts.processor)) + + # dict of [vendor][code] : Avp + avp_codes = collections.defaultdict(dict) + + # dict of [vendor][name] : Avp + avp_names = collections.defaultdict(dict) + + # Process files + for filename in args: + avpproc.filename(filename) + with open(filename, 'r') as csvfile: + csvdata = csv.DictReader(csvfile, CSV_COLUMN_NAMES, + restkey='extra_cells', restval='') + standard = '' + vendor = 0 + errors = [] + for row in csvdata: + try: + if csvdata.restkey in row: + raise ValueError('Extra cells: {}'.format( + ','.join(row[csvdata.restkey]))) + if row['name'] in (None, '', 'Attribute Name'): + continue + elif row['name'].startswith('#'): + comment = row['name'][1:] + avpproc.comment(comment, filename, csvdata.line_num) + elif row['name'].startswith('@'): + parameter = row['name'][1:] + value = row['code'] + if False: + pass + elif parameter == 'standard': + standard = value + elif parameter == 'vendor': + vendor = int(value) + else: + raise ValueError('Unknown parameter "{}"'.format( + parameter)) + avpproc.parameter(parameter, value) + else: + avp = Avp(filename=filename, line_num=csvdata.line_num, + standard=standard, vendor=vendor, + **row) + # Ensure AVP vendor/code not already defined + if avp.code in avp_codes[avp.vendor]: + conflict = avp_codes[avp.vendor][avp.code] + avp_conflict('Code', avp, conflict) + avp_codes[avp.vendor][avp.code] = avp + # Ensure AVP vendor/name not already defined + if avp.name in avp_names[avp.vendor]: + conflict = avp_names[avp.vendor][avp.name] + avp_conflict('Name', avp, conflict) + avp_names[avp.vendor][avp.name] = avp + # Process AVP + avpproc.avp(avp) + except (TypeError, ValueError) as e: + errors.append('CSV file "{}" line {}: {}\n'.format( + filename, csvdata.line_num, e)) + if errors: + sys.stderr.write(''.join(errors)) + sys.exit(1) + + # Generate result + avpproc.generate() + + +if __name__ == '__main__': + main() + +# vim: set et sw=4 sts=4 : |