substitute unicode general punctuation left/right single/double quotes
This commit is contained in:
@ -673,6 +673,18 @@ std::string utils::utf8_subst(const std::string &line)
|
|||||||
// ZERO WIDTH SPACE (U+200B)
|
// ZERO WIDTH SPACE (U+200B)
|
||||||
if ((in[i] == 0xe2U) && (in[i+1] == 0x80U) && (in[i+2] == 0x8bU))
|
if ((in[i] == 0xe2U) && (in[i+1] == 0x80U) && (in[i+2] == 0x8bU))
|
||||||
out += ' ', i += 2;
|
out += ' ', i += 2;
|
||||||
|
// LEFT SINGLE QUOTATION MARK (U+2018)
|
||||||
|
if ((in[i] == 0xe2U) && (in[i+1] == 0x80U) && (in[i+2] == 0x98U))
|
||||||
|
out += '\'', i += 2;
|
||||||
|
// RIGHT SINGLE QUOTATION MARK (U+2019)
|
||||||
|
if ((in[i] == 0xe2U) && (in[i+1] == 0x80U) && (in[i+2] == 0x99U))
|
||||||
|
out += '\'', i += 2;
|
||||||
|
// LEFT DOUBLE QUOTATION MARK (U+201C)
|
||||||
|
if ((in[i] == 0xe2U) && (in[i+1] == 0x80U) && (in[i+2] == 0x9cU))
|
||||||
|
out += '"', i += 2;
|
||||||
|
// RIGHT DOUBLE QUOTATION MARK (U+201D)
|
||||||
|
if ((in[i] == 0xe2U) && (in[i+1] == 0x80U) && (in[i+2] == 0x9dU))
|
||||||
|
out += '"', i += 2;
|
||||||
// NARROW NO-BREAK SPACE (U+202F)
|
// NARROW NO-BREAK SPACE (U+202F)
|
||||||
if ((in[i] == 0xe2U) && (in[i+1] == 0x80U) && (in[i+2] == 0xafU))
|
if ((in[i] == 0xe2U) && (in[i+1] == 0x80U) && (in[i+2] == 0xafU))
|
||||||
out += ' ', i += 2;
|
out += ' ', i += 2;
|
||||||
|
|||||||
Reference in New Issue
Block a user