Feat/include nomenclatures in csv parser

.github/workflows/generate-model.yaml

@@ -47,6 +47,16 @@ jobs:
         with:
           python-version: '3.11'
 
+      - name: Install nomenclature_parser python requirements
+        working-directory: ./nomenclature_parser
+        if: steps.filter.outputs.parsing_required == 'true'
+        run: pip install -r ./requirements.txt
+
+      - name: Run nomenclature_parser
+        working-directory: ./nomenclature_parser
+        if: steps.filter.outputs.parsing_required == 'true'
+        run: python nomenclature_parser.py
+
       - name: Install python requirements
         working-directory: ./csv_parser
         if: steps.filter.outputs.parsing_required == 'true' || steps.filter.outputs.test_case_parsing_required == 'true'

csv_parser/README.md

@@ -7,13 +7,13 @@ For the UML generator (run automatically within the CSV parser), in addition to
 # Usage
 ## CSV Parser
 ```bash
-# Params to specify the sheet and version
+# -s/--sheet, -n/--name, -m/--model-type, -p/--filepath are required
+# -v/--version defaults to today (YY.MM.DD), -f/--filter defaults to False
 # by default, integrate uml generation process
-python csv_parser.py -s RC-DE -v 0.5  
-python csv_parser.py --sheet RC-DE --version 0.5
-# Defaults to RC-EDA and today (YY.MM.DD)
-python csv_parser.py
+python csv_parser.py -s RC-DE -n toto -v 0.5 -m distributionElement -p models/model.xlsx
+python csv_parser.py --sheet RC-DE --name toto --version 0.5 --model-type distributionElement --filepath models/model.xlsx
 ```
+`--model-type` (rootElement) and `--filepath` (source xlsx) are looked up per sheet in `out/schemas.yaml`.
 
 ## Full pipeline
 The full pipeline is defined in the GitHub action. 

csv_parser/csv_parser.py

@@ -97,9 +97,9 @@ def get_params_from_sheet(sheet):
             'perimeterColumns': perimeter_columns
         }
 
-    def get_nomenclature(elem):
+    def get_nomenclature(elem, col='Détails de format'):
         # filename to target (.csv format)
-        nomenclature_name = elem['Détails de format'][elem['Détails de format'].index(':')+1:].strip()
+        nomenclature_name = elem[col][elem[col].index(':')+1:].strip()
         path_file = ''
         nomenclature_files = os.listdir(os.path.join("..", "nomenclature_parser", "out", "latest", "csv"))
         file_extension = ".csv"
@@ -118,7 +118,7 @@ def get_nomenclature(elem):
             print(f'Known nomenclatures are {nomenclature_files}')
             print("Check if some nomenclature files disappeared. If so, last run of nomenclature_parser.py likely failed.")
             exit(1)
-        return L_ret
+        return nomenclature_name, L_ret
 
     params = get_params_from_sheet(sheet)
     # Schema name is in name = RC-EDA (or RS-EDA) for instance
@@ -145,7 +145,7 @@ def is_allowing_additional_properties(name):
     # Column validation
     REQUIRED_COLUMNS = [
         *[f"Donnée (Niveau {i})" for i in range(1, DATA_DEPTH + 1)],
-        'ID', 'Description', 'Exemples', 'Balise', 'Cardinalité', 'Objet', 'Format (ou type)', 'Détails de format',
+        'ID', 'Description', 'Exemples', 'Balise', 'Cardinalité', 'Objet', 'Format (ou type)', 'Détails de format', 'Nomenclature',
         *([perimeter_filter] if perimeter_filter else [])
     ]
     if not (set(REQUIRED_COLUMNS) <= set(df.columns)):
@@ -212,12 +212,12 @@ def find_data_level(element):
                 return i
         return 0
 
-    def format_codeandlabel_properties(child, parent):
-        code_file = parent['Détails de format']
-        """ For 'Code', set code file name to the 'Détails de format' column, remove it from parent """
+    def format_codeandlabel_properties(child, parent, col='Détails de format'):
+        code_file = parent[col]
+        """ For 'Code', set code file name to the column, remove it from parent """
         if child['Balise'] == "code":
-            child['Détails de format'] = code_file
-            df.loc[parent.ID-1, 'Détails de format'] = 'nan'
+            child[col] = code_file
+            df.loc[parent.ID-1, col] = 'nan'
         """Set the level of the child to be the level of the parent + 1"""
         if find_data_level(child) != find_data_level(parent)+1:
             child = shift_child_data_levels(child, parent)
@@ -256,7 +256,9 @@ def regenerate_ids(df):
                     # the level of data of each property to be equal to the row's level + 1
                     prop_cpy = first_codeandlabel_properties[i - 1].copy()
                     prop_cpy['ID'] = row['ID']+i/10
-                    df.loc[index + i/10] = format_codeandlabel_properties(prop_cpy, row)
+                    prop_cpy = format_codeandlabel_properties(prop_cpy, row)
+                    prop_cpy = format_codeandlabel_properties(prop_cpy, row, col='Nomenclature')
+                    df.loc[index + i/10] = prop_cpy
 
     df.sort_index(axis=0, ascending=True, inplace=True, kind='quicksort')
     df.reset_index(drop=True, inplace=True)
@@ -527,9 +529,9 @@ def build_example(elem):
         'additionalProperties': is_allowing_additional_properties(name)
     }
 
-    def has_format_details(elem, details):
+    def has_format_details(elem, details, col='Détails de format'):
         """Does elem have a format details starting with details?"""
-        return str(elem['Détails de format']) != 'nan' and elem['Détails de format'].startswith(details)
+        return str(elem[col]) != 'nan' and elem[col].startswith(details)
 
     def type_matching(child):
         """Get the matching type for a given type name"""
@@ -541,8 +543,9 @@ def type_matching(child):
         elif typeName == 'phoneNumber':
             return 'string', r'^tel:([#\+\*]|37000|00+)?[0-9]{2,15}$', None
         else:
-            if has_format_details(child, FormatFlags.REGEX):
-                return typeName, child['Détails de format'][child['Détails de format'].index(':')+1:].strip(), None
+            col = 'Détails de format'
+            if has_format_details(child, FormatFlags.REGEX, col):
+                return typeName, child[col][child[col].index(':')+1:].strip(), None
             else:
                 return typeName, None, None
 
@@ -579,7 +582,14 @@ def add_field_child_property(parent, child, definitions):
             childDetails['enum'] = child['Détails de format'][child['Détails de format'].index(':')+1:].strip().split(', ')
         # key word nomenclature trigger search over nomenclature folder for matching file
         if has_format_details(child, FormatFlags.NOMENCLATURE):
-            childDetails['enum'] = get_nomenclature(child)
+            nomenclature_name, nomenclature_codes = get_nomenclature(child)
+            childDetails['x-nomenclature'] = nomenclature_name
+            childDetails['enum'] = nomenclature_codes
+        if has_format_details(child, FormatFlags.NOMENCLATURE, "Nomenclature"):
+            nomenclature_name, _ = get_nomenclature(child, "Nomenclature")
+            if 'x-nomenclature' in childDetails:
+                print(f"{Color.ORANGE}WARNING: Field 'Nomenclature' defined twice for child '{child['full_name']}', overwritting nomenclature with {nomenclature_name}")
+            childDetails['x-nomenclature'] = nomenclature_name
         properties = definitions['properties']
         if is_array(child):
             properties[child['name']] = {
@@ -1020,6 +1030,8 @@ def def_to_table(name, definition, title='', doc=None, style='Medium Shading 1 A
     parser.add_argument('-n', '--name', required=True, help='The name to be given to the schema folder/file')
     parser.add_argument('-v', '--version', help='The version number to be used in model. Defaults to today.')
     parser.add_argument('-f', '--filter', default=False, help='If present, only 15-18 fields will be kept')
+    parser.add_argument('-m', '--model-type', required=True, help='The rootElement name for this schema (see out/schemas.yaml).')
+    parser.add_argument('-p', '--filepath', required=True, help='Path to the Excel file containing the sheet (e.g. models/model.xlsx).')
     args = parser.parse_args()
 
-    run(args.sheet, args.name, args.version, args.filter)
+    run(args.sheet, args.name, args.version, args.filter, args.model_type, args.filepath)

csv_parser/json_schema2xsd/src/main/resources/schemas.yaml

@@ -1,28 +1,4 @@
 schemas:
-- automaticGeneration: Y
-  customExtendClass: null
-  customExtendPackage: null
-  file: model-technical.xlsx
-  header: Y
-  package: technical
-  perimeter: TECHNICAL
-  rootElement: technical
-  schema: TECHNICAL
-  sheet: TECHNICAL
-  subschema: N
-  xmlns: eda:1.9:technical
-- automaticGeneration: Y
-  customExtendClass: ContentMessage
-  customExtendPackage: com.hubsante.model.edxl
-  file: model-technical.xlsx
-  header: N
-  package: technical.noreq
-  perimeter: TECHNICAL_NOREQ
-  rootElement: technicalNoreq
-  schema: TECHNICAL_NOREQ
-  sheet: TECHNICAL
-  subschema: N
-  xmlns: eda:1.9:technicalNoreq
 - automaticGeneration: Y
   customExtendClass: ContentMessage
   customExtendPackage: com.hubsante.model.edxl
@@ -275,3 +251,27 @@ schemas:
   sheet: customContent
   subschema: N
   xmlns: cisu:3.0
+- automaticGeneration: Y
+  customExtendClass: null
+  customExtendPackage: null
+  file: model-technical.xlsx
+  header: Y
+  package: technical
+  perimeter: TECHNICAL
+  rootElement: technical
+  schema: TECHNICAL
+  sheet: TECHNICAL
+  subschema: N
+  xmlns: eda:1.9:technical
+- automaticGeneration: Y
+  customExtendClass: ContentMessage
+  customExtendPackage: com.hubsante.model.edxl
+  file: model-technical.xlsx
+  header: N
+  package: technical.noreq
+  perimeter: TECHNICAL_NOREQ
+  rootElement: technicalNoreq
+  schema: TECHNICAL_NOREQ
+  sheet: TECHNICAL
+  subschema: N
+  xmlns: eda:1.9:technicalNoreq