Finetuning was done using MedNER with the following script:
We limited the training to the following languages: English, Dutch, French, German, Spanish, Italian, Danish, Swedish and Portuguese, which are relatively close, in a linguistic sense, allowing for a languages to benefit from cross-over information.
The model was train in a multiclass-sense, using a cross-entropy loss. We replaced the standard 768-weight linear layer by 3x768 dense layers with 10% dropout and ReLu activations.
#!/usr/bin/env bash
set -euo pipefail
# Ready-to-run training script for:
# ai4privacy/pii-masking-openpii-1.5m
#
# Usage:
# bash scripts/run_hf_ai4privacy_training.sh multilabel
# bash scripts/run_hf_ai4privacy_training.sh multiclass
#
# Optional:
# export HF_TOKEN=hf_xxx # only needed for gated/private datasets
MODE="${1:-multiclass}"
COMMON_ARGS=(
--lang multi
--hf_dataset ai4privacy/pii-masking-openpii-1.5m
--text_column source_text
--tags_column privacy_mask
--selector_column language
--selection nl en fr de es it da sv pt
--parse_annotations
--train_model
--force_splitter
--model FacebookAI/xlm-roberta-base
--chunk_size=32
--max_token_length=32
--batch_size 2
--accumulation_steps 8
--learning_rate=2e-5
--weight_decay=1e-5
--num_epochs=5
--num_splits=5
--use_class_weights
--classifier_hidden_layers 768 768 768
--chunk_type paragraph
--entity_types AGE DATE BUILDINGNUM CITY ZIPCODE STREET GIVENNAME SURNAME SOCIALNUM PASSPORTNUM SEX GENDER TELEPHONENUM IDCARDNUM EMAIL
)
if [ -n "${HF_TOKEN:-}" ]; then
COMMON_ARGS+=(--hf_token "${HF_TOKEN}")
fi
if [ "${MODE}" = "multiclass" ]; then
poetry run python -m cardioner.main \
"${COMMON_ARGS[@]}" \
--multiclass \
--output_dir /out/put/location_multilabel
elif [ "${MODE}" = "multilabel" ]; then
poetry run python -m cardioner.main \
"${COMMON_ARGS[@]}" \
--output_dir /out/put/location_multilabel
else
echo "Unknown mode: ${MODE}. Use 'multilabel' or 'multiclass'." >&2
exit 1
fi
Token classification scores (answering the question: given the span, to which class does it belong?):
"eval_AGE": {
"f1": 0.9672360768551506,
"number": 32192,
"precision": 0.9484920871902061,
"recall": 0.9867358349900597
},
"eval_BUILDINGNUM": {
"f1": 0.9832519792331442,
"number": 28889,
"precision": 0.9798886138613861,
"recall": 0.9866385129287965
},
"eval_CITY": {
"f1": 0.9726022504018575,
"number": 41669,
"precision": 0.9651668399659703,
"recall": 0.9801531114257601
},
"eval_DATE": {
"f1": 0.997746454524687,
"number": 80940,
"precision": 0.9972108062744517,
"recall": 0.9982826785273041
},
"eval_EMAIL": {
"f1": 0.9962614462862082,
"number": 51445,
"precision": 0.9953818689847873,
"recall": 0.9971425794537856
},
"eval_GENDER": {
"f1": 0.974448482073879,
"number": 17457,
"precision": 0.973473587925909,
"recall": 0.9754253308128544
},
"eval_GIVENNAME": {
"f1": 0.9365210222588624,
"number": 100350,
"precision": 0.9334706161888168,
"recall": 0.9395914299950174
},
"eval_IDCARDNUM": {
"f1": 0.9049785748995008,
"number": 22068,
"precision": 0.8827889338964061,
"recall": 0.9283124886713794
},
"eval_PASSPORTNUM": {
"f1": 0.9242263349514563,
"number": 12959,
"precision": 0.9087180252069506,
"recall": 0.9402731692260206
},
"eval_SEX": {
"f1": 0.9694227341286166,
"number": 14570,
"precision": 0.9683604985618408,
"recall": 0.970487302676733
},
"eval_SOCIALNUM": {
"f1": 0.9280586914587484,
"number": 15327,
"precision": 0.9004663721158567,
"recall": 0.9573954459450643
},
"eval_STREET": {
"f1": 0.9810103469263542,
"number": 32803,
"precision": 0.9793116019078287,
"recall": 0.9827149955796726
},
"eval_SURNAME": {
"f1": 0.9306129607436204,
"number": 85192,
"precision": 0.926976695433423,
"recall": 0.9342778664663348
},
"eval_TELEPHONENUM": {
"f1": 0.9940678080618354,
"number": 36941,
"precision": 0.9924450920079866,
"recall": 0.9956958393113343
},
"eval_ZIPCODE": {
"f1": 0.9856799457604832,
"number": 24346,
"precision": 0.9860648661980516,
"recall": 0.9852953257208577
}
End2End scores (answering the question: given a text, what are the spans and to which class do they belong?):
{
"strict": {
"per_category": {
"AGE": {
"Precision": 0.86,
"Recall": 0.87,
"F1": 0.87
},
"BUILDINGNUM": {
"Precision": 0.94,
"Recall": 0.91,
"F1": 0.93
},
"CITY": {
"Precision": 0.88,
"Recall": 0.88,
"F1": 0.88
},
"DATE": {
"Precision": 0.69,
"Recall": 0.68,
"F1": 0.68
},
"EMAIL": {
"Precision": 0.12,
"Recall": 0.1,
"F1": 0.11
},
"GENDER": {
"Precision": 0.95,
"Recall": 0.88,
"F1": 0.91
},
"GIVENNAME": {
"Precision": 0.92,
"Recall": 0.9,
"F1": 0.91
},
"IDCARDNUM": {
"Precision": 0.77,
"Recall": 0.64,
"F1": 0.7
},
"PASSPORTNUM": {
"Precision": 0.84,
"Recall": 0.74,
"F1": 0.79
},
"SEX": {
"Precision": 0.95,
"Recall": 0.9,
"F1": 0.92
},
"SOCIALNUM": {
"Precision": 0.71,
"Recall": 0.82,
"F1": 0.76
},
"STREET": {
"Precision": 0.86,
"Recall": 0.87,
"F1": 0.86
},
"SURNAME": {
"Precision": 0.91,
"Recall": 0.89,
"F1": 0.9
},
"TELEPHONENUM": {
"Precision": 0.67,
"Recall": 0.73,
"F1": 0.7
},
"ZIPCODE": {
"Precision": 0.91,
"Recall": 0.89,
"F1": 0.9
}
},
"micro": {
"Precision": 0.79,
"Recall": 0.77,
"F1": 0.78
},
"macro": {
"Precision": 0.8,
"Recall": 0.78,
"F1": 0.79
}
},
"relaxed": {
"per_category": {
"AGE": {
"Precision": 0.9,
"Recall": 0.9,
"F1": 0.9
},
"BUILDINGNUM": {
"Precision": 0.96,
"Recall": 0.92,
"F1": 0.94
},
"CITY": {
"Precision": 0.94,
"Recall": 0.95,
"F1": 0.95
},
"DATE": {
"Precision": 0.97,
"Recall": 0.96,
"F1": 0.96
},
"EMAIL": {
"Precision": 0.96,
"Recall": 0.79,
"F1": 0.86
},
"GENDER": {
"Precision": 0.97,
"Recall": 0.9,
"F1": 0.93
},
"GIVENNAME": {
"Precision": 0.98,
"Recall": 0.97,
"F1": 0.97
},
"IDCARDNUM": {
"Precision": 0.9,
"Recall": 0.74,
"F1": 0.81
},
"PASSPORTNUM": {
"Precision": 0.92,
"Recall": 0.8,
"F1": 0.86
},
"SEX": {
"Precision": 0.95,
"Recall": 0.91,
"F1": 0.93
},
"SOCIALNUM": {
"Precision": 0.78,
"Recall": 0.9,
"F1": 0.84
},
"STREET": {
"Precision": 0.95,
"Recall": 0.96,
"F1": 0.96
},
"SURNAME": {
"Precision": 0.97,
"Recall": 0.96,
"F1": 0.97
},
"TELEPHONENUM": {
"Precision": 0.91,
"Recall": 0.98,
"F1": 0.94
},
"ZIPCODE": {
"Precision": 0.94,
"Recall": 0.92,
"F1": 0.93
}
},
"micro": {
"Precision": 0.95,
"Recall": 0.92,
"F1": 0.93
},
"macro": {
"Precision": 0.93,
"Recall": 0.9,
"F1": 0.92
}
}
}
which we obtained by running:
MODEL=/path/to/model
ARGS=(
--inference_only
--model_path="$MODEL"
--corpus_inference=ai4privacy/pii-masking-openpii-1.5m
--text_column=source_text
--tags_column=privacy_mask
--lang=multi
--inference_stride=32
--selector_column=language
--selection nl en fr de es it da sv pt
--inference_pipe=dt4h
--trust_remote_code
--output_file_prefix=multiclass_multilang_
--allow_numeric_tags AGE BUILDINGNUM DATE IDCARDNUM PASSPORTNUM SOCIALNUM TELEPHONENUM ZIPCODE
)
poetry run python -m cardioner.main "${ARGS[@]}"