Note that the code examples from different sections are not necessarily linked to one another.
Examples#
instances.json examples#
instances.json - first example#
{
"Instances": [
{
"ClassName": "Demo.Declaration",
"PropertyName": "Code"
},
{
"ClassName": "Demo.Declaration",
"PropertyName": "UidDeclaration"
},
{
"ClassName": "Demo.Declaration",
"PropertyName": "Description",
"ExtractRegex": "!~[A-Za-z0-9/=\\.\\+_]+?~|(\\w(?<!_)|\\~)+"
},
{
"ClassName": "Demo.Declaration",
"PropertyName": "DateDeclaration",
"Type": "DateTime",
"Format": "MM/dd/yyyy HH:mm:ss"
},
{
"ClassName": "Demo.Declaration",
"PropertyName": "UidParent"
},
{
"ClassName": "Demo.Event",
"PropertyName": "Label"
},
{
"ClassName": "Demo.Event",
"PropertyName": "DateEvent",
"Type": "DateTime",
"Format": "MM/dd/yyyy HH:mm:ss"
},
{
"ClassName": "Demo.Event",
"PropertyName": "Type"
},
{
"ClassName": "Demo.Event",
"PropertyName": "NumberEvent"
},
{
"ClassName": "Demo.Event",
"PropertyName": "Code"
}
]
}instances.json - second example#
{
"Instances": [
{
"ClassName": "Demo.users",
"PropertyName": "first_name"
},
{
"ClassName": "Demo.users",
"PropertyName": "last_name"
},
{
"ClassName": "Demo.users",
"PropertyName": "phone_number",
"ExtractValueRegex": "[0-9]+"
},
{
"ClassName": "Demo.users",
"PropertyName": "fonction_metier_id",
"Type": "Long"
},
{
"ClassName": "Demo.incidents",
"PropertyName": "date_de_debut",
"Type": "DateTime",
"Format": "dd/MM/yy",
"AllowedFormats": [
"dd/MM/yyyy",
"dd/MM/yy",
"dd/M/yyyy",
"dd/M/yy",
"d/MM/yyyy",
"d/MM/yy",
"d/M/yyyy",
"d/M/yy"
]
},
{
"ClassName": "Demo.incidents",
"PropertyName": "date_de_debut_only_time",
"Type": "DateTime",
"Format": "HH:mm",
"AllowedFormats": [
"H:mm"
]
},
{
"ClassName": "Demo.incidents",
"PropertyName": "date_de_fin",
"Type": "DateTime",
"Format": "dd/MM/yy",
"AllowedFormats": [
"dd/MM/yyyy",
"dd/MM/yy",
"dd/M/yyyy",
"dd/M/yy",
"d/MM/yyyy",
"d/MM/yy",
"d/M/yyyy",
"d/M/yy"
]
},
{
"ClassName": "Demo.incidents",
"PropertyName": "location"
}
]
}rightsContexts.json examples#
rightsContexts.json - first example#
{
"RightsContexts": {
"DataProcessor": {
"Evidences": [
{
"Name": "Rights",
"Value": "Full"
}
]
},
"NotDeleteRightsContext": {
"Evidences": [
{
"Name": "Rights",
"Value": "Read"
}
]
}
}
}rightsContexts.json - second example#
{
"RightsContexts": {
"Transform": {
"Evidences": [
{
"Name": "Right",
"Value": "Transform"
}
]
}
}
}processingContexts.json examples#
processingContexts.json - first example#
{
"ProcessingContexts": {
"Deprotection": {
"Evidences": [
{
"Name": "Action",
"Value": "Deprotect"
},
{
"Name": "Transformation",
"Value": "Clear"
}
]
},
"Protection": {
"Evidences": [
{
"Name": "Action",
"Value": "Protect"
}
]
}
}
}processingContexts.json - second example#
{
"ProcessingContexts": {
"Protection": {
"Evidences": [
{
"Name": "Action",
"Value": "Protect"
}
]
},
"ParticularProtection": {
"Evidences": [
{
"Name": "Action",
"Value": "CustomProtect"
}
]
},
"Deprotection": {
"Evidences": [
{
"Name": "Action",
"Value": "Deprotect"
}
]
}
}
}dataProcessor.json examples#
Processing templates#
To ease the configuration of a Files Processor where the transformation is often the same, one can define ProcessingTemplates at the job level and then call ProcessingTemplateName from the concerned Columns at file level. ProcessingTemplates takes as parameters Name to call the ProcessingTemplates and Processing to set the processing steps.
Processing Template - example of declaration#
{
"ProcessingTemplates": [
{
"Name": "ProtectionTemplate",
"Processing": [
{
"Step": 1,
"Processors": [
{
"Name": "Default",
"Type": "RPSEngineTransformation"
}
]
}
]
}
]
}Processing Template - example of usage#
{
"Name": "test_file",
"InputFile": {
"Path": "C:/RPSFilesProcessorData/Project/test_file.csv"
},
"OutputFile": {
"Path": "C:/RPSFilesProcessorData/Project/protected_test_file.csv"
},
"ErrorFile": {
"Path": "C:/RPSFilesProcessorData/Project/error_error_file.csv"
},
"Columns": [
{
"Names": [
"column1"
],
"ClassName": "class.name1",
"PropertyName": "property1",
"ProcessingTemplateName": "ProtectionTemplate"
},
{
"Names": [
"column2"
],
"ClassName": "class.name2",
"PropertyName": "property2",
"ProcessingTemplateName": "ProtectionTemplate"
}
]
}References between files#
To refer to another file's column you need to declare Relations array for the current file. The array contains the fields File for the referenced file, and Keys, itself with fields SourceColumn (source column) and DestColumn (destination column).
References between files - example#
{
"Relations": [
{
"File": "INCIDENTS",
"Keys": {
"SourceColumn": "ID",
"DestColumn": "INCIDENT_ID"
}
}
]
}ForeignKey parameter#
Used in relation to option Relations declaration in another file, ForeignKey is used in the destination file to declare the link.
ForeignKey parameter - example#
{
"Name": "INCIDENTS",
"InputFile": {
"Path": "incidents.csv"
},
"OutputFile": {
"Path": "protected_incidents.csv"
},
"BatchSizeByRows": 1000,
"ForeignKey": "ID",
"Columns": [
{
"Names": [
"DATE_DE_DEBUT"
],
"ClassName": "Demo.incidents",
"PropertyName": "date_de_debut",
"Processing": [
{
"Step": 1,
"Processors": [
{
"Name": "ParseDateTime",
"Type": "Preprocessing"
}
]
}
]
}
]
}Batch sizes#
Batches are used in order to group and send values for transformation to RPS Engine by some grouped
number. It is used to optimize performances by a faster processing of files on the RPS Engine and an
increase of the speed of communication between RPS Files Processor and RPS Engine.
BatchSizeByRows parameter can be set on job level (in the encoding, rightContext, processingContext section) will then be applied to all transformations and for all files. If set on file level, it will only be applied to transformations in chosen file.
Beware, the value of this parameter must be an int number.
Batch sizes - example#
{
"Name": "INCIDENTS",
"InputFile": {
"Path": "incidents.csv"
},
"OutputFile": {
"Path": "protected_incidents.csv"
},
"BatchSizeByRows": 1000,
"ForeignKey": "ID",
"Columns": [
{
"Names": [
"DATE_DE_DEBUT"
],
"ClassName": "Demo.incidents",
"PropertyName": "date_de_debut",
"Processing": [
{
"Step": 1,
"Processors": [
{
"Name": "ParseDateTime",
"Type": "Preprocessing"
}
]
},
{
"Step": 2,
"Processors": [
{
"Name": "Default",
"Type": "RPSEngineTransformation",
"Dependencies": {
"min": "{STEP1.INCIDENTS.DATE_DE_DEBUT}-90d",
"max": "{DATETIME}"
}
}
]
}
]
}
]
}Conditional processor#
It can be useful do use conditions while processing a document, and these conditions can be complexified by
using the logical operator AND and OR. The operators are defined using && and ||
in the condition, as shown below.
Conditional processor, usage of AND and OR in conditions - example#
{
"Processors": [
{
"Name": "Conditional",
"Type": "RPSEngineTransformation",
"Options": {
"If": {
"Condition": "{fileName.columnName}==$(0) || {fileName.columnName}==$(1)",
"Action": {
"RightsContext": "Transform",
"ProcessingContext": "Protect"
}
},
"Else": {
"Action": {
"RightsContext": "Read"
}
}
}
}
]
}Value expressions#
Option Value used in Processors definition, can contain an expression or a simple value defined by text. The expression is used for variables, defined between brackets "{" and "}". Three variables are predefined:
| Variable | Description |
|---|---|
| {DATETIME} | Current date and time |
| {DATE} | Current date |
| {TIME} | Current time |
Date and Time variables may be changed using expressions "+" or "-" a number of days "d", hours "h",
minutes "m", or seconds "s". You can also use variables to create a link to another Column for
example by using {STEP_NO.FILE_NAME.COLUMN_NAME} where
STEP_NOis an optional parameter (default value is STEP0 or data from the original file if not specified),FILE_NAMEis a required parameter that takes the value of the Name field of the aimed file,COLUMN_NAMEis a required parameter that takes the value of the Column field in the aimed file.
See below for a preview.
Simple value - example#
"Value": "Test value"Expression - example#
{
"Processors": [
{
"Name": "ToLowerCaseValue",
"Type": "Postprocessing",
"Value": "{STEP_NO.FILE_NAME.COLUMN_NAME}+7d"
}
]
}Date and Time variables - examples#
| Example | Description |
|---|---|
| {DATETIME}-90d | Current date time minus 90 days |
| {STEP1.INCIDENTS.DATE_DE_DEBUT}+7d | Add 7 days for Value from Step 1 of DATE_DE_DEBUT column from INCIDENTS file |
| {CAPACITE.DATE}>={DATE}+2d | Used for conditional expressions. If the value from Step 0 (original value) of DATE column from INCIDENTS file more or equal current date plus 2 days then = True, else = False |
| {STEP1.INCIDENTS.DATE_DE_DEBUT}AND{STEP2.ENTREES.CREATED_AT} | If AND used for dates then choose more newest date. Max of (STEP1.INCIDENTS.DATE_DE_DEBUT, STEP2.ENTREES.CREATED_AT). |
| !{STEP1.INCIDENTS.DATE_DE_DEBUT} | Used for conditional expressions. If the value from Step 1 of DATE_DE_DEBUT column from INCIDENTS file is null or empty then = True, else = False |
Other options#
Other options may be set on job level, for instance HeaderRowsCount or HeaderRowsIndex that can be changed for CSV files with headers on multiple lines. HeaderRowsCount defines the number of rows in the file while HeaderRowsIndex provides the number of the row to use as input for the name of columns. Option SkipEmptyRecords is used to ignore empty columns or fields in a CSV without throwing an error and stopping the protection of the rest of the file.
Complete dataProcessor.json configurations examples#
dataProcessor.json - first example#
{
"Jobs": [
{
"Name": "csv_protect",
"TypeName": "CsvFilesProcessorJob",
"CronExpression": "0 0/1 * 1/1 * ? *",
"Options": {
"CsvOptions": {
"Delimiter": ";",
"DetectColumnCountChanges": true
},
"Encoding": "windows-1252",
"RightsContext": "DataProcessor",
"ProcessingContext": "Protection",
"BatchSizeByRows": 400,
"Files": [
{
"Name": "DECLARATIONS",
"InputFile": {
"Path": "Declarations.csv"
},
"OutputFile": {
"Path": "Declarations_protected.csv"
},
"Columns": [
{
"Names": [
"UIDDECLARATION"
],
"ClassName": "Demo.Declaration",
"PropertyName": "UidDeclaration",
"Processing": [
{
"Step": 1,
"Processors": [
{
"Name": "Default",
"Type": "RPSEngineTransformation"
}
]
}
]
},
{
"Names": [
"DATEDECLARATION"
],
"ClassName": "Demo.Declaration",
"PropertyName": "DateDeclaration",
"Processing": [
{
"Step": 1,
"Processors": [
{
"Name": "Default",
"Type": "RPSEngineTransformation"
}
]
}
]
},
{
"Names": [
"DESCRIPTION"
],
"ClassName": "Demo.Declaration",
"PropertyName": "Description",
"Processing": [
{
"Step": 1,
"Processors": [
{
"Name": "Default",
"Type": "RPSEngineTransformation"
}
]
}
]
},
{
"Names": [
"UIDPARENT"
],
"ClassName": "Demo.Declaration",
"PropertyName": "UidParent",
"Processing": [
{
"Step": 1,
"Processors": [
{
"Name": "Default",
"Type": "RPSEngineTransformation"
}
]
}
]
},
{
"Names": [
"CODE"
],
"ClassName": "Demo.Declaration",
"PropertyName": "Code",
"Processing": [
{
"Step": 1,
"Processors": [
{
"Name": "Default",
"Type": "RPSEngineTransformation"
}
]
}
]
}
]
},
{
"Name": "EVENT",
"InputFile": {
"Path": "Events.csv"
},
"OutputFile": {
"Path": "Events_protected.csv"
},
"Columns": [
{
"Names": [
"LABEL"
],
"ClassName": "Demo.Event",
"PropertyName": "Label",
"Processing": [
{
"Step": 1,
"Processors": [
{
"Name": "Default",
"Type": "RPSEngineTransformation"
}
]
}
]
},
{
"Names": [
"DATEEVENT"
],
"ClassName": "Demo.Event",
"PropertyName": "DateEvent",
"Processing": [
{
"Step": 1,
"Processors": [
{
"Name": "Default",
"Type": "RPSEngineTransformation"
}
]
}
]
},
{
"Names": [
"TYPE"
],
"ClassName": "Demo.Event",
"PropertyName": "Type",
"Processing": [
{
"Step": 1,
"Processors": [
{
"Name": "Default",
"Type": "RPSEngineTransformation"
}
]
}
]
},
{
"Names": [
"NUMBEREVENT"
],
"ClassName": "Demo.Event",
"PropertyName": "NumberEvent",
"Processing": [
{
"Step": 1,
"Processors": [
{
"Name": "Default",
"Type": "RPSEngineTransformation"
}
]
}
]
},
{
"Names": [
"CODE"
],
"ClassName": "Demo.Event",
"PropertyName": "Code",
"Processing": [
{
"Step": 1,
"Processors": [
{
"Name": "Default",
"Type": "RPSEngineTransformation"
}
]
}
]
}
]
}
]
}
},
{
"Name": "csv_deprotect",
"TypeName": "CsvFilesProcessorJob",
"CronExpression": "0 0/1 * 1/1 * ? *",
"Options": {
"CsvOptions": {
"Delimiter": ";",
"DetectColumnCountChanges": true
},
"Encoding": "windows-1252",
"RightsContext": "DataProcessor",
"ProcessingContext": "Deprotection",
"BatchSizeByRows": 400,
"Files": [
{
"Name": "DECLARATIONS",
"InputFile": {
"Path": "Declarations_protected.csv"
},
"OutputFile": {
"Path": "Declarations_deprotected.csv"
},
"Columns": [
{
"Names": [
"DESCRIPTION"
],
"ClassName": "Demo.Declaration",
"PropertyName": "Description",
"Processing": [
{
"Step": 1,
"Processors": [
{
"Name": "Default",
"Type": "RPSEngineTransformation"
}
]
}
]
},
{
"Names": [
"CODE"
],
"ClassName": "Demo.Declaration",
"PropertyName": "Code",
"Processing": [
{
"Step": 1,
"Processors": [
{
"Name": "Default",
"Type": "RPSEngineTransformation"
}
]
}
]
}
]
},
{
"Name": "EVENTS",
"InputFile": {
"Path": "Events_protected.csv"
},
"OutputFile": {
"Path": "Events_deprotected.csv"
},
"Columns": [
{
"Names": [
"LABEL"
],
"ClassName": "Demo.Event",
"PropertyName": "Label",
"Processing": [
{
"Step": 1,
"Processors": [
{
"Name": "Default",
"Type": "RPSEngineTransformation"
}
]
}
]
},
{
"Names": [
"NUMBEREVENT"
],
"ClassName": "Demo.Event",
"PropertyName": "NumberEvent",
"Processing": [
{
"Step": 1,
"Processors": [
{
"Name": "Default",
"Type": "RPSEngineTransformation"
}
]
}
]
}
]
}
]
}
}
]
}dataProcessor.json - second example#
{
"Jobs": [
{
"Name": "project_csv_protect",
"TypeName": "CsvFilesProcessorJob",
"CronExpression": "0 0/1 * 1/1 * ? *",
"Options": {
"CsvOptions": {
"Delimiter": ",",
"DetectColumnCountChanges": true
},
"Encoding": "iso-8859-1",
"RightsContext": "Transform",
"ProcessingContext": "Protect",
"BatchSizeByRows": 400,
"HeaderRowsCount": 2,
"HeaderRowIndex": 1,
"ProcessingTemplates": [
{
"Name": "ProtectionTemplate",
"Processing": [
{
"Step": 1,
"Processors": [
{
"Name": "Default",
"Type": "RPSEngineTransformation"
}
]
}
]
}
],
"Files": [
{
"Names": [
"Employees-1st-file",
"Employees-2nd-file"
],
"InputFile": {
"Path": "RPSFilesProcessorData/ProjectName/in/{FILE_NAME}.csv"
},
"OutputFile": {
"Path": "RPSFilesProcessorData/ProjectName/out/protected_{FILE_NAME}.csv"
},
"ErrorFile": {
"Path": "RPSFilesProcessorData/ProjectName/errors/error_{FILE_NAME}.csv"
},
"Columns": [
{
"Names": [
"first-name"
],
"ClassName": "Employee",
"PropertyName": "Names",
"ProcessingTemplateName": "ProtectionTemplate"
},
{
"Names": [
"last-name"
],
"ClassName": "Employee",
"PropertyName": "Names",
"ProcessingTemplateName": "ProtectionTemplate"
},
{
"Names": [
"marital-status"
],
"ClassName": "Employee",
"PropertyName": "MaritalStatus",
"ProcessingTemplateName": "ProtectionTemplate"
},
{
"Names": [
"preferred-name"
],
"ClassName": "Employee",
"PropertyName": "PreferredFirstName",
"ProcessingTemplateName": "ProtectionTemplate"
},
{
"Names": [
"formal-name"
],
"ClassName": "Employee",
"PropertyName": "PreferredLastName",
"ProcessingTemplateName": "ProtectionTemplate"
}
]
},
{
"Name": "Marital_Status",
"InputFile": {
"Path": "RPSFilesProcessorData/ProjectName/in/Marital_Status.csv"
},
"OutputFile": {
"Path": "RPSFilesProcessorData/ProjectName/out/protected_Marital_Status.csv"
},
"ErrorFile": {
"Path": "RPSFilesProcessorData/ProjectName/errors/error_Marital_Status.csv"
},
"Columns": [
{
"Names": [
"values.label.defaultValue"
],
"ClassName": "Employee",
"PropertyName": "MaritalStatus",
"ProcessingTemplateName": "ProtectionTemplate"
},
{
"Names": [
"values.label.en_GB"
],
"ClassName": "Employee",
"PropertyName": "MaritalStatus",
"ProcessingTemplateName": "ProtectionTemplate"
},
{
"Names": [
"values.label.fr_FR"
],
"ClassName": "Employee",
"PropertyName": "MaritalStatus",
"ProcessingTemplateName": "ProtectionTemplate"
}
]
},
{
"Name": "Supported_Languages",
"InputFile": {
"Path": "RPSFilesProcessorData/ProjectName/in/Supported_Languages.csv"
},
"OutputFile": {
"Path": "RPSFilesProcessorData/ProjectName/out/protected_Supported_Languages.csv"
},
"ErrorFile": {
"Path": "RPSFilesProcessorData/ProjectName/errors/error_Supported_Languages.csv"
},
"Columns": [
{
"Names": [
"externalName"
],
"ClassName": "Employee",
"PropertyName": "PreferredMailingLanguage",
"ProcessingTemplateName": "ProtectionTemplate"
}
]
}
]
}
}
]
}