#!/usr/bin/python
# -*- coding: utf-8 -*-
#
# created on: 2013jun05
# created by: Markus W. Scherer
"""Converts CLDR collation files from XML syntax to ICU syntax.
Handles the CLDR collation data in the post-CLDR 23 trunk in 2013 June.
Preserves indentation (except where it joins lines) and text vs. NCR etc.
Does not handle arbitrary LDML XML collation syntax."""
# Invoke with two arguments:
# - the source folder path
# - the destination folder path
# For example:
# ~/svn.cldr$ collicu/tools/scripts/coll2icu.py trunk/common/collation collicu/common/collation
import codecs
import glob
import os.path
import sys
def GetIndent(s):
for i in range(len(s)):
if s[i] not in " \t": return s[:i]
return s
# substring replacements
replacements = (
# White space and syntax characters must be quoted.
# Using '\\u0020' rather than just ' ' for clarity.
("<reset> </reset>", "&'\\u0020'"), # can't just replace all "> <"
(">!<", ">'!'<"),
('>"<', ">'\\\"'<"),
(">"<", ">'\\\"'<"),
(">#<", ">'\\u0023'<"),
(">$<", ">'$'<"),
(">%<", ">'%'<"),
(">&<", ">'&'<"),
(">&<", ">'&'<"),
(">'<", ">''<"),
(">'<", ">''<"),
(">(<", ">'('<"),
(">)<", ">')'<"),
(">*<", ">'*'<"),
(">+<", ">'+'<"),
(">,<", ">','<"),
(">-<", ">'-'<"),
(">.<", ">'.'<"),
(">/<", ">'/'<"),
(">:<", ">':'<"),
(">;<", ">';'<"),
("><<", ">'<'<"),
(">=<", ">'='<"),
(">><", ">'>'<"),
(">?<", ">'?'<"),
(">@<", ">'@'<"),
(">[<", ">'['<"),
(">\\<", ">'\\\\'<"),
(">]<", ">']'<"),
(">^<", ">'^'<"),
(">_<", ">'_'<"),
(">`<", ">'`'<"),
(">{<", ">'{'<"),
(">|<", ">'|'<"),
(">}<", ">'}'<"),
(">~<", ">'~'<"),
# ha.xml has the following
("'y", "''y"),
("'Y", "''Y"),
# kl.xml has the following
("K'", "K''"),
# not Pattern_White_Space, just obscure
(u"\u00A0", u"\\u00A0"),
(u"\u200C", u"\\u200C"),
(u"\u200D", u"\\u200D"),
(u"\u3000", u"\\u3000"),
# obscure, and some tools do not handle noncharacters well
(u"\uFDD0", u"'\\uFDD0'"),
# The old ICU collation rule parser seems to need more escaping than it should.
(u"≠", u"'≠'"),
# fi.xml resets contain a space
(u" ̵</reset>", u"'\\u0020'̵"),
# fa.xml <sc> with non-NFD_Inert chars
(u"<sc>\u0650\u064f\u064b\u064d\u064c</sc>", u"<<\u0650<<\u064f<<\u064b<<\u064d<<\u064c"),
# ml.xml strings contain spaces
(u" </s>", u"'\\u0020'"),
(u" </reset>", u"'\\u0020'"),
# vi.xml <sc> with non-NFD_Inert chars
(u"<sc>\u0309\u0303\u0301\u0323</sc>", u"<<\u0309<<\u0303<<\u0301<<\u0323"),
# en_US_POSIX needs a lot of quoting.
("<pc> !"#$%&'()*+,-./</pc>", "<*'\\u0020'-'/'"),
("<pc>0123456789:;<=>?@</pc>", "<*0-'@'"),
("<pc>[\]^_`</pc>", "<*'['-'`'"),
("<pc>{|}~</pc>", "<*'{'-'\u007F'"),
# CJK parenthesized resets
("<reset>(", "&'('"),
(")</reset>", "')'"),
# Convert XML elements into ICU syntax.
("><!--", "> #"), # add a space before an inline comment
("<!--", "#"),
(" -->", ""),
("-->", ""),
("<reset>", "&"),
('<reset before="primary">', "&[before 1]"),
('<reset before="secondary">', "&[before 2]"),
('<reset before="tertiary">', "&[before 3]"),
("</reset>", ""),
("<p>", "<"),
("</p>", ""),
("<s>", "<<"),
("</s>", ""),
("<t>", "<<<"),
("</t>", ""),
("<i>", "="),
("</i>", ""),
("<pc>", "<*"),
("</pc>", ""),
("<sc>", "<<*"),
("</sc>", ""),
("<tc>", "<<<*"),
("</tc>", ""),
("<ic>", "=*"),
("</ic>", ""),
("<x>", ""),
("</x>", ""),
("<extend>", "/"),
("</extend>", ""),
("</context>", "|"),
("<first_tertiary_ignorable/>", "[first tertiary ignorable]"),
("<last_tertiary_ignorable/>", "[last tertiary ignorable]"),
("<first_secondary_ignorable/>", "[first secondary ignorable]"),
("<last_secondary_ignorable/>", "[last secondary ignorable]"),
("<first_primary_ignorable/>", "[first primary ignorable]"),
("<last_primary_ignorable/>", "[last primary ignorable]"),
("<first_variable/>", "[first variable]"),
("<last_variable/>", "[last variable]"),
("<first_non_ignorable/>", "[first regular]"),
("<last_non_ignorable/>", "[last regular]"),
("<last_non_ignorable />", "[last regular]"),
("<first_trailing/>", "[first trailing]"),
("<last_trailing/>", "[last trailing]")
)
def ConvertFile(src, dest):
in_rules = False
partial = ""
in_ml_comment = False
for line in src:
if "<rules>" in line:
indent = GetIndent(line)
stripped = line.strip()
# Replace import-only rules with import elements.
if stripped == '<rules><import source="sr"/></rules>':
dest.write(indent + '<import source="sr"/>\n')
elif stripped == '<rules><import source="hr" type="search"/></rules>':
dest.write(indent + '<import source="hr" type="search"/>\n')
elif stripped == '<rules><import source="hr"/></rules>':
dest.write(indent + '<import source="hr"/>\n')
elif stripped == '<rules><import source="ps"/></rules>':
dest.write(indent + '<import source="ps"/>\n')
else:
# Replace the XML <rules> section with ICU syntax rules in <cr>.
assert stripped == "<rules>"
dest.write(indent + "<cr><![CDATA[\n")
in_rules = True
elif "</rules>" in line:
# Flush, and go back to just copying lines until the next <rules>.
if partial:
dest.write(partial + "\n")
partial = ""
in_ml_comment = False
dest.write(GetIndent(line) + "]]></cr>\n")
in_rules = False
else:
if in_rules:
# Find out whether we want to concatenate the current line
# with the previous and/or next one.
finish_partial = False # Finish collected, partial input.
start_ml_comment = False # Start of a multi-line comment.
stop_comment = False # End of a comment, must terminate the line.
if ("<reset" in line) or line.lstrip().startswith("<!--"):
finish_partial = True
if partial and len(partial.strip()) > 80:
finish_partial = True
if "<!--" in line and "-->" not in line:
start_ml_comment = True
if "-->" in line:
assert line.rstrip().endswith("-->")
stop_comment = True
# Convert XML syntax to ICU syntax.
if "<context>" in line:
# Swap context & relation:
# <x><context>カ</context><i>ー</i></x>
# turns into
# =カ|ー
if "<i>" in line:
line = line.replace("<i>", "").replace("<context>", "<i>")
elif "<t>" in line:
line = line.replace("<t>", "").replace("<context>", "<t>")
for (xml, icu) in replacements:
line = line.replace(xml, icu)
while True:
# Convert a Numeric Character Reference to \\uhhhh.
i = line.find("&#x")
if i < 0: break
limit = line.find(";", i + 3)
cp = line[i + 3:limit]
while len(cp) < 4: cp = "0" + cp
assert len(cp) == 4 # not handling supplementary code points
line = line[:i] + "\\u" + cp + line[limit + 1:]
# Start/continue/finish concatenation, and output.
if partial and finish_partial:
# Write collected input.
dest.write(partial + "\n")
partial = ""
if start_ml_comment:
# Start a multi-line comment.
assert not partial
comment_indent = GetIndent(line) # can be the empty string
in_ml_comment = True
elif in_ml_comment:
# Continue a multi-line comment.
assert not partial
if line.startswith(comment_indent):
if line[len(comment_indent)] in " \t":
# Preserve further indentation.
line = comment_indent + "#" + line[len(comment_indent):]
else:
# Add a space after the #.
line = comment_indent + "# " + line[len(comment_indent):]
else:
# Indent at least as much as the first line.
line = line.lstrip()
if line:
line = comment_indent + "# " + line
else:
line = comment_indent + "#\n"
elif stop_comment:
# Just output the line, do not start collecting input.
# ICU-syntax comments end with the end of the line,
# do not append rules to them.
if partial:
line = partial + line.lstrip() + "\n"
partial = ""
elif not partial:
# Start collecting input.
partial = line.rstrip()
elif partial:
# Continue collecting input.
partial += line.strip()
if stop_comment:
in_ml_comment = False
if not partial: dest.write(line)
def main():
(src_root, dest_root) = sys.argv[1:3]
src_pattern = os.path.join(src_root, "*.xml")
for src_path in glob.iglob(src_pattern):
basename = os.path.basename(src_path)
dest_path = os.path.join(dest_root, basename)
with codecs.open(src_path, "r", "UTF-8") as src:
with codecs.open(dest_path, "w", "UTF-8") as dest:
ConvertFile(src, dest)
if __name__ == "__main__":
main()