searchtag # rextended utf8 to ucs2, utf8 to ucs2 pdu
Without using tables, converting one UTF-8 string to one UCS-2 string (unicode 2 bytes entry point), but obviously only the 0x0000 to 0xFFFF characters supported to the UCS-2.
On error or with unsupported characters, the replacement character 0xFF 0xFE is used.
:global UTF8toUCS2 do={
:local repch "\FF\FD"
:if ([:typeof $2] = "no-replace") do={:set repch ""}
:local numbyte2hex do={
:local input [:tonum $1]
:local hexchars "0123456789ABCDEF"
:local convert [:pick $hexchars (($input >> 4) & 0xF)]
:set convert ($convert.[:pick $hexchars ($input & 0xF)])
:return $convert
}
:local charsString ""
:for x from=0 to=15 step=1 do={ :for y from=0 to=15 step=1 do={
:local tmpHex "$[:pick "0123456789ABCDEF" $x ($x+1)]$[:pick "0123456789ABCDEF" $y ($y+1)]"
:set $charsString "$charsString$[[:parse "(\"\\$tmpHex\")"]]"
} }
:local chr2int do={
:if (($1="") or ([:len $1] > 1) or ([:typeof $1] = "nothing")) do={:return -1}
:return [:find $2 $1 -1]
}
:local string $1
:if (([:typeof $string] != "str") or ($string = "")) do={ :return "" }
:local output ""
:local lenstr [:len $string]
:local read1; :local char1; :local char2; :local char3; :local char4; :local ucsvalue
:local outstr ""
:local pos 0
:while ($pos < $lenstr) do={
:set read1 [:pick $string $pos ($pos + 1)]
:set char1 [$chr2int $read1 $charsString]
:if ($char1 < 0x80) do={
:set outstr "\00$read1"
}
:if ((($char1 > 0x7F) and ($char1 < 0xC2)) or ($char1 > 0xEF)) do={
:set outstr $repch
}
:set char2 [$chr2int [:pick $string ($pos + 1) ($pos + 2)] $charsString]
:if (($char1 > 0xC1) and ($char1 < 0xE0)) do={
:if (($char2 < 0x80) or ($char2 > 0xBF)) do={
:set outstr $repch
} else={
:set ucsvalue ((($char1 - 0xC0) * 0x40) + ($char2 - 0x80))
:set outstr "$[:pick $charsString (($ucsvalue >> 8) & 0xFF)]$[:pick $charsString ($ucsvalue & 0xFF)]"
:set pos ($pos + 1)
}
}
:set char3 [$chr2int [:pick $string ($pos + 2) ($pos + 3)] $charsString]
:if (($char1 > 0xDF) and ($char1 < 0xF0)) do={
:if ((($char2 < 0x80) or ($char2 > 0xBF)) \
or ((($char1 = 0xE0) and ($char2 < 0xA0)) or (($char1 = 0xED) and ($char2 > 0x9F)))) do={
:set outstr $repch
} else={
:if (($char3 < 0x80) or ($char3 > 0xBF)) do={
:set outstr $repch
:set pos ($pos + 1)
} else={
:set ucsvalue ((($char1 - 0xE0) * 0x1000) + (($char2 - 0x80) * 0x40) + ($char3 - 0x80))
:set outstr "$[:pick $charsString (($ucsvalue >> 8) & 0xFF)]$[:pick $charsString ($ucsvalue & 0xFF)]"
:set pos ($pos + 2)
}
}
}
# the following commented lines are not used on UCS-2
# but I have already prepared my script for future changes to work with all UNICODE code points from 0x000000 to 0x10FFFF as well...
# :set char4 [$chr2int [:pick $string ($pos + 3) ($pos + 4)] $charsString]
# :if (($char1 > 0xEF) and ($char1 < 0xF5)) do={
# :if ((($char2 < 0x80) or ($char2 > 0xBF)) \
# or ((($char1 = 0xF0) and ($char2 < 0x90)) or (($char1 = 0xF4) and ($char2 > 0x8F)))) do={
# :set outstr $repch
# } else={
# :if (($char3 < 0x80) or ($char3 > 0xBF)) do={
# :set outstr $repch
# :set pos ($pos + 1)
# } else={
# :if (($char4 < 0x80) or ($char4 > 0xBF)) do={
# :set outstr $repch
# :set pos ($pos + 2)
# } else={
# :set ucsvalue ((($char1 - 0xF0) * 0x40000) + (($char2 - 0x80) * 0x1000) + \
# (($char3 - 0x80) * 0x40) + ($char4 - 0x80))
# :set outstr "$[:pick $charsString (($ucsvalue >> 16) & 0xFF)]"
# :set outstr "$outstr$[:pick $charsString (($ucsvalue >> 8) & 0xFF)]$[:pick $charsString ($ucsvalue & 0xFF)]"
# :set pos ($pos + 3)
# }
# }
# }
# }
:set output "$output$outstr"
:set pos ($pos + 1)
}
:return $output
}
For example, for convert the string "hello my friend camión Ññ" on UTF-8 to UCS-2:
{
:global testutf8 "\68\65\6C\6C\6F\20\6D\79\20\66\72\69\65\6E\64\20\63\61\6D\69\C3\B3\6E\20\C3\91\C3\B1"
:global testucs2 [$UTF8toUCS2 $testutf8]
:put $testutf8
:put $testucs2
/sys scri env pri
}
hello my friend cami n
hello my friend cami n
NAME VALUE
x testucs2 \00h\00e\00l\00l\00o\00 \00m\00y\00 \00f\00r\00i\00e\00n\00d\00 \00c\00a\00m\00i\00\F3\00n\00 \00\D1\00\F1
x testutf8 hello my friend cami\C3\B3n \C3\91\C3\B1
MikroTik can not display non-7-bit-ASCII characters on terminal, but on memory are present correct values.
Entry points: ó = 00 FE, Ñ = 00 D1,ñ = 00 F1
And this is for obtain a HEX string to send message on SMS usable by AT commands for calculate UCS-2 PDU:
:global UTF8toUCS2hexstring do={
:local repch "FFFD"
:if ([:typeof $2] = "no-replace") do={:set repch ""}
:local numbyte2hex do={
:local input [:tonum $1]
:local hexchars "0123456789ABCDEF"
:local convert [:pick $hexchars (($input >> 4) & 0xF)]
:set convert ($convert.[:pick $hexchars ($input & 0xF)])
:return $convert
}
:local charsString ""
:for x from=0 to=15 step=1 do={ :for y from=0 to=15 step=1 do={
:local tmpHex "$[:pick "0123456789ABCDEF" $x ($x+1)]$[:pick "0123456789ABCDEF" $y ($y+1)]"
:set $charsString "$charsString$[[:parse "(\"\\$tmpHex\")"]]"
} }
:local chr2int do={
:if (($1="") or ([:len $1] > 1) or ([:typeof $1] = "nothing")) do={:return -1}
:return [:find $2 $1 -1]
}
:local string $1
:if (([:typeof $string] != "str") or ($string = "")) do={ :return "" }
:local output ""
:local lenstr [:len $string]
:local read1; :local char1; :local char2; :local char3; :local char4; :local ucsvalue
:local outstr ""
:local pos 0
:while ($pos < $lenstr) do={
:set read1 [:pick $string $pos ($pos + 1)]
:set char1 [$chr2int $read1 $charsString]
:if ($char1 < 0x80) do={
:set outstr "00$[$numbyte2hex $char1]"
}
:if ((($char1 > 0x7F) and ($char1 < 0xC2)) or ($char1 > 0xEF)) do={
:set outstr $repch
}
:set char2 [$chr2int [:pick $string ($pos + 1) ($pos + 2)] $charsString]
:if (($char1 > 0xC1) and ($char1 < 0xE0)) do={
:if (($char2 < 0x80) or ($char2 > 0xBF)) do={
:set outstr $repch
} else={
:set ucsvalue ((($char1 - 0xC0) * 0x40) + ($char2 - 0x80))
:set outstr "$[$numbyte2hex (($ucsvalue >> 8) & 0xFF)]$[$numbyte2hex ($ucsvalue & 0xFF)]"
:set pos ($pos + 1)
}
}
:set char3 [$chr2int [:pick $string ($pos + 2) ($pos + 3)] $charsString]
:if (($char1 > 0xDF) and ($char1 < 0xF0)) do={
:if ((($char2 < 0x80) or ($char2 > 0xBF)) \
or ((($char1 = 0xE0) and ($char2 < 0xA0)) or (($char1 = 0xED) and ($char2 > 0x9F)))) do={
:set outstr $repch
} else={
:if (($char3 < 0x80) or ($char3 > 0xBF)) do={
:set outstr $repch
:set pos ($pos + 1)
} else={
:set ucsvalue ((($char1 - 0xE0) * 0x1000) + (($char2 - 0x80) * 0x40) + ($char3 - 0x80))
:set outstr "$[$numbyte2hex (($ucsvalue >> 8) & 0xFF)]$[$numbyte2hex ($ucsvalue & 0xFF)]"
:set pos ($pos + 2)
}
}
}
# the following commented lines are not used on UCS-2
# but I have already prepared my script for future changes to work with all UNICODE code points from 0x000000 to 0x10FFFF as well...
# :set char4 [$chr2int [:pick $string ($pos + 3) ($pos + 4)] $charsString]
# :if (($char1 > 0xEF) and ($char1 < 0xF5)) do={
# :if ((($char2 < 0x80) or ($char2 > 0xBF)) \
# or ((($char1 = 0xF0) and ($char2 < 0x90)) or (($char1 = 0xF4) and ($char2 > 0x8F)))) do={
# :set outstr $repch
# } else={
# :if (($char3 < 0x80) or ($char3 > 0xBF)) do={
# :set outstr $repch
# :set pos ($pos + 1)
# } else={
# :if (($char4 < 0x80) or ($char4 > 0xBF)) do={
# :set outstr $repch
# :set pos ($pos + 2)
# } else={
# :set ucsvalue ((($char1 - 0xF0) * 0x40000) + (($char2 - 0x80) * 0x1000) + \
# (($char3 - 0x80) * 0x40) + ($char4 - 0x80))
# :set outstr "$[$numbyte2hex (($ucsvalue >> 16) & 0xFF)]"
# :set outstr "$outstr$[$numbyte2hex (($ucsvalue >> 8) & 0xFF)]$[$numbyte2hex ($ucsvalue & 0xFF)]"
# :set pos ($pos + 3)
# }
# }
# }
# }
:set output "$output$outstr"
:set pos ($pos + 1)
}
:return $output
}
On this example the string "hello my friend camión Ññ" on UTF-8 is converted on UCS-2 for use on SMS PDU "AT" command:
{
:global testutf8 "\68\65\6C\6C\6F\20\6D\79\20\66\72\69\65\6E\64\20\63\61\6D\69\C3\B3\6E\20\C3\91\C3\B1"
:global testucs2 [$UTF8toUCS2hexstring $testutf8]
:put $testutf8
:put $testucs2
/sys scri env pri
}
hello my friend cami n
NAME VALUE
x testucs2 00680065006C006C006F0020006D007900200066007200690065006E0064002000630061006D006900F3006E002000D100F1
x testutf8 hello my friend cami\C3\B3n \C3\91\C3\B1
The
(32)00680065006C006C006F0020006D007900200066007200690065006E0064002000630061006D006900F3006E002000D100F1
is the encoded message on UCS-2 for create the SMS PDU. This message is 25 characters, but use 50 characters on the SMS for the UCS-2 PDU encoding.
On PDU 0x32 (50) must be added at the start, is the length of the message, and on UCS-2 is never a odd number.