@InProceedings{BenjiaZhou2023,
author="Benjia Zhou
and Zhigang Chen
and Albert Clapes
and Jun Wan
and Yanyan Liang
and Sergio Escalera
and Zhen Lei
and Du Zhang",
title="Gloss-free Sign Language Translation: Improving from Visual-Language Pretraining",
booktitle="IEEE/CVF International Conference on Computer Vision (ICCV) Workshops",
year="2023",
abstract="Sign Language Translation (SLT) is a challenging task due to its cross-domain nature, involving the translation of visual-gestural language to text. Many previous methods employ an intermediate representation, i.e., gloss sequences, to facilitate SLT, thus transforming it into a two-stage task of sign language recognition (SLR) followed by sign language translation (SLT). However, the scarcity of gloss-annotated sign language data, combined with the information bottleneck in the mid-level gloss representation, has hindered the further development of the SLT task. To address this challenge, we propose a novel Gloss-Free SLT based on Visual-Language Pretraining (GFSLT-VLP), which improves SLT by inheriting language-oriented prior knowledge from pre-trained models, without any gloss annotation assistance. Our approach involves two stages: (i) integrating Contrastive Language-Image Pre-training (CLIP) with masked self-supervised learning to create pre-tasks that bridge the semantic gap between visual and textual representations and restore masked sentences, and (ii) constructing an end-to-end architecture with an encoder-decoder-like structure that inherits the parameters of the pre-trained Visual Encoder and Text Decoder from the first stage. The seamless combination of these novel designs forms a robust sign language representation and significantly improves gloss-free sign language translation. In particular, we have achieved unprecedented improvements in terms of BLEU-4 score on the PHOENIX14T dataset (>+5) and the CSL-Daily dataset (>+3) compared to state-of-the-art gloss-free SLT methods. Furthermore, our approach also achieves competitive results on the PHOENIX14T dataset when compared with most of the gloss-based methods.",
optnote="HUPBA;",
optnote="exported from refbase (http://158.109.8.37/show.php?record=3839), last updated on Thu, 18 Jan 2024 10:08:37 +0100",
doi="10.48550/arXiv.2307.14768",
opturl="https://arxiv.org/abs/2307.14768",
file=":http://158.109.8.37/files/ZCC2023.pdf:PDF"
}