sts

Running on A100

App Files Files Community

sts / uroman /lib /NLP /Romanizer.pm

multimodalart HF staff

First commit

7bcf8d7 over 1 year ago

raw

history blame contribute delete

93 kB

	################################################################
	# #
	# Romanizer #
	# #
	################################################################

	package NLP::Romanizer;

	use NLP::Chinese;
	use NLP::UTF8;
	use NLP::utilities;
	use JSON;
	$utf8 = NLP::UTF8;
	$util = NLP::utilities;
	$chinesePM = NLP::Chinese;

	my $verbosePM = 0;
	%empty_ht = ();

	my $braille_capital_letter_indicator = "\xE2\xA0\xA0";
	my $braille_number_indicator = "\xE2\xA0\xBC";
	my $braille_decimal_point = "\xE2\xA0\xA8";
	my $braille_comma = "\xE2\xA0\x82";
	my $braille_solidus = "\xE2\xA0\x8C";
	my $braille_numeric_space = "\xE2\xA0\x90";
	my $braille_letter_indicator = "\xE2\xA0\xB0";
	my $braille_period = "\xE2\xA0\xB2";

	sub new {
	local($caller) = @_;

	my $object = {};
	my $class = ref( $caller ) \|\| $caller;
	bless($object, $class);
	return $object;
	}

	sub load_unicode_data {
	local($this, *ht, $filename) = @_;
	# ../../data/UnicodeData.txt

	$n = 0;
	if (open(IN, $filename)) {
	while (<IN>) {
	if (($unicode_value, $char_name, $general_category, $canon_comb_classes, $bidir_category, $char_decomp_mapping, $decimal_digit_value, $digit_value, $numeric_value, $mirrored, $unicode_1_0_name, $comment_field, $uc_mapping, $lc_mapping, $title_case_mapping) = split(";", $_)) {
	$utf8_code = $utf8->unicode_hex_string2string($unicode_value);
	$ht{UTF_TO_CHAR_NAME}->{$utf8_code} = $char_name;
	$ht{UTF_NAME_TO_UNICODE}->{$char_name} = $unicode_value;
	$ht{UTF_NAME_TO_CODE}->{$char_name} = $utf8_code;
	$ht{UTF_TO_CAT}->{$utf8_code} = $general_category;
	$ht{UTF_TO_NUMERIC}->{$utf8_code} = $numeric_value unless $numeric_value eq "";
	$n++;
	}
	}
	close(IN);
	# print STDERR "Loaded $n entries from $filename\n";
	} else {
	print STDERR "Can't open $filename\n";
	}
	}

	sub load_unicode_overwrite_romanization {
	local($this, *ht, $filename) = @_;
	# ../../data/UnicodeDataOverwrite.txt

	$n = 0;
	if (open(IN, $filename)) {
	while (<IN>) {
	next if /^#/;
	$unicode_value = $util->slot_value_in_double_colon_del_list($_, "u");
	$romanization = $util->slot_value_in_double_colon_del_list($_, "r");
	$numeric = $util->slot_value_in_double_colon_del_list($_, "num");
	$picture = $util->slot_value_in_double_colon_del_list($_, "pic");
	$syllable_info = $util->slot_value_in_double_colon_del_list($_, "syllable-info");
	$tone_mark = $util->slot_value_in_double_colon_del_list($_, "tone-mark");
	$char_name = $util->slot_value_in_double_colon_del_list($_, "name");
	$entry_processed_p = 0;
	$utf8_code = $utf8->unicode_hex_string2string($unicode_value);
	if ($unicode_value) {
	$ht{UTF_TO_CHAR_ROMANIZATION}->{$utf8_code} = $romanization if $romanization;
	$ht{UTF_TO_NUMERIC}->{$utf8_code} = $numeric if defined($numeric) && ($numeric ne "");
	$ht{UTF_TO_PICTURE_DESCR}->{$utf8_code} = $picture if $picture;
	$ht{UTF_TO_SYLLABLE_INFO}->{$utf8_code} = $syllable_info if $syllable_info;
	$ht{UTF_TO_TONE_MARK}->{$utf8_code} = $tone_mark if $tone_mark;
	$ht{UTF_TO_CHAR_NAME}->{$utf8_code} = $char_name if $char_name;
	$entry_processed_p = 1 if $romanization \|\| $numeric \|\| $picture \|\| $syllable_info \|\| $tone_mark;
	}
	$n++ if $entry_processed_p;
	}
	close(IN);
	} else {
	print STDERR "Can't open $filename\n";
	}
	}

	sub load_script_data {
	local($this, *ht, $filename) = @_;
	# ../../data/Scripts.txt

	$n = 0;
	if (open(IN, $filename)) {
	while (<IN>) {
	next unless $script_name = $util->slot_value_in_double_colon_del_list($_, "script-name");
	$abugida_default_vowel_s = $util->slot_value_in_double_colon_del_list($_, "abugida-default-vowel");
	$alt_script_name_s = $util->slot_value_in_double_colon_del_list($_, "alt-script-name");
	$language_s = $util->slot_value_in_double_colon_del_list($_, "language");
	$direction = $util->slot_value_in_double_colon_del_list($_, "direction"); # right-to-left
	$font_family_s = $util->slot_value_in_double_colon_del_list($_, "font-family");
	$ht{SCRIPT_P}->{$script_name} = 1;
	$ht{SCRIPT_NORM}->{(uc $script_name)} = $script_name;
	$ht{DIRECTION}->{$script_name} = $direction if $direction;
	foreach $language (split(/,\s*/, $language_s)) {
	$ht{SCRIPT_LANGUAGE}->{$script_name}->{$language} = 1;
	$ht{LANGUAGE_SCRIPT}->{$language}->{$script_name} = 1;
	}
	foreach $alt_script_name (split(/,\s*/, $alt_script_name_s)) {
	$ht{SCRIPT_NORM}->{$alt_script_name} = $script_name;
	$ht{SCRIPT_NORM}->{(uc $alt_script_name)} = $script_name;
	}
	foreach $abugida_default_vowel (split(/,\s*/, $abugida_default_vowel_s)) {
	$ht{SCRIPT_ABUDIGA_DEFAULT_VOWEL}->{$script_name}->{$abugida_default_vowel} = 1 if $abugida_default_vowel;
	}
	foreach $font_family (split(/,\s*/, $font_family_s)) {
	$ht{SCRIPT_FONT}->{$script_name}->{$font_family} = 1 if $font_family;
	}
	$n++;
	}
	close(IN);
	# print STDERR "Loaded $n entries from $filename\n";
	} else {
	print STDERR "Can't open $filename\n";
	}
	}

	sub unicode_hangul_romanization {
	local($this, $s, $pass_through_p) = @_;

	$pass_through_p = 0 unless defined($pass_through_p);
	@leads = split(/\s+/, "g gg n d dd r m b bb s ss - j jj c k t p h");
	# @vowels = split(/\s+/, "a ae ya yai e ei ye yei o oa oai oi yo u ue uei ui yu w wi i");
	@vowels = split(/\s+/, "a ae ya yae eo e yeo ye o wa wai oe yo u weo we wi yu eu yi i");
	@tails = split(/\s+/, "- g gg gs n nj nh d l lg lm lb ls lt lp lh m b bs s ss ng j c k t p h");
	$result = "";
	@chars = $utf8->split_into_utf8_characters($s, "return only chars", *empty_ht);
	foreach $char (@chars) {
	$unicode = $utf8->utf8_to_unicode($char);
	if (($unicode >= 0xAC00) && ($unicode <= 0xD7A3)) {
	$code = $unicode - 0xAC00;
	$lead_index = int($code / (28*21));
	$vowel_index = int($code/28) % 21;
	$tail_index = $code % 28;
	$rom = $leads[$lead_index] . $vowels[$vowel_index] . $tails[$tail_index];
	$rom =~ s/-//g;
	$result .= $rom;
	} elsif ($pass_through_p) {
	$result .= $char;
	}
	}
	return $result;
	}

	sub listify_comma_sep_string {
	local($this, $s) = @_;

	@result_list = ();
	return @result_list unless $s =~ /\S/;
	$s = $util->trim2($s);
	my $elem;

	while (($elem, $rest) = ($s =~ /^("(?:\\"\|[^"])"\|'(?:\\'\|[^'])'\|[^"', ]+),\s(.)$/)) {
	push(@result_list, $util->dequote_string($elem));
	$s = $rest;
	}
	push(@result_list, $util->dequote_string($s)) if $s =~ /\S/;

	return @result_list;
	}

	sub braille_string_p {
	local($this, $s) = @_;

	return ($s =~ /^(\xE2[\xA0-\xA3][\x80-\xBF])+$/);
	}

	sub register_word_boundary_info {
	local($this, *ht, $lang_code, $utf8_source_string, $utf8_target_string, $use_only_for_whole_word_p,
	$use_only_at_start_of_word_p, $use_only_at_end_of_word_p,
	$dont_use_at_start_of_word_p, $dont_use_at_end_of_word_p) = @_;

	if ($use_only_for_whole_word_p) {
	if ($lang_code) {
	$ht{USE_ONLY_FOR_WHOLE_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = 1;
	} else {
	$ht{USE_ONLY_FOR_WHOLE_WORD}->{$utf8_source_string}->{$utf8_target_string} = 1;
	}
	}
	if ($use_only_at_start_of_word_p) {
	if ($lang_code) {
	$ht{USE_ONLY_AT_START_OF_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = 1;
	} else {
	$ht{USE_ONLY_AT_START_OF_WORD}->{$utf8_source_string}->{$utf8_target_string} = 1;
	}
	}
	if ($use_only_at_end_of_word_p) {
	if ($lang_code) {
	$ht{USE_ONLY_AT_END_OF_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = 1;
	} else {
	$ht{USE_ONLY_AT_END_OF_WORD}->{$utf8_source_string}->{$utf8_target_string} = 1;
	}
	}
	if ($dont_use_at_start_of_word_p) {
	if ($lang_code) {
	$ht{DONT_USE_AT_START_OF_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = 1;
	} else {
	$ht{DONT_USE_AT_START_OF_WORD}->{$utf8_source_string}->{$utf8_target_string} = 1;
	}
	}
	if ($dont_use_at_end_of_word_p) {
	if ($lang_code) {
	$ht{DONT_USE_AT_END_OF_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = 1;
	} else {
	$ht{DONT_USE_AT_END_OF_WORD}->{$utf8_source_string}->{$utf8_target_string} = 1;
	}
	}
	}

	sub load_romanization_table {
	local($this, *ht, $filename) = @_;
	# ../../data/romanization-table.txt

	$n = 0;
	$line_number = 0;
	if (open(IN, $filename)) {
	while (<IN>) {
	$line_number++;
	next if /^#/;
	if ($_ =~ /^::preserve\s/) {
	$from_unicode = $util->slot_value_in_double_colon_del_list($_, "from");
	$to_unicode = $util->slot_value_in_double_colon_del_list($_, "to");
	if ($from_unicode =~ /^(?:U\+\|\\u)[0-9A-F]{4,}$/i) {
	$from_unicode =~ s/^(?:U\+\|\\u)//;
	$from_code_point = hex($from_unicode);
	} else {
	$from_code_point = "";
	}
	if ($to_unicode =~ /^(?:U\+\|\\u)[0-9A-F]{4,}$/i) {
	$to_unicode =~ s/^(?:U\+\|\\u)//;
	$to_code_point = hex($to_unicode);
	} else {
	$to_code_point = $from_code_point;
	}
	if ($from_code_point ne "") {
	# print STDERR "Preserve code-points $from_unicode--$to_unicode = $from_code_point--$to_code_point\n";
	foreach $code_point (($from_code_point .. $to_code_point)) {
	$utf8_string = $utf8->unicode2string($code_point);
	$ht{UTF_CHAR_MAPPING}->{$utf8_string}->{$utf8_string} = 1;
	}
	$n++;
	}
	next;
	}
	$utf8_source_string = $util->slot_value_in_double_colon_del_list($_, "s");
	$utf8_target_string = $util->slot_value_in_double_colon_del_list($_, "t");
	$utf8_alt_target_string_s = $util->slot_value_in_double_colon_del_list($_, "t-alt");
	$use_alt_in_pointed_p = ($_ =~ /::use-alt-in-pointed\b/);
	$use_only_for_whole_word_p = ($_ =~ /::use-only-for-whole-word\b/);
	$use_only_at_start_of_word_p = ($_ =~ /::use-only-at-start-of-word\b/);
	$use_only_at_end_of_word_p = ($_ =~ /::use-only-at-end-of-word\b/);
	$dont_use_at_start_of_word_p = ($_ =~ /::dont-use-at-start-of-word\b/);
	$dont_use_at_end_of_word_p = ($_ =~ /::dont-use-at-end-of-word\b/);
	$use_only_in_lower_case_enviroment_p = ($_ =~ /::use-only-in-lower-case-enviroment\b/);
	$word_external_punctuation_p = ($_ =~ /::word-external-punctuation\b/);
	$utf8_source_string =~ s/\s*$//;
	$utf8_target_string =~ s/\s*$//;
	$utf8_alt_target_string_s =~ s/\s*$//;
	$utf8_target_string =~ s/^"(.*)"$/$1/;
	$utf8_target_string =~ s/^'(.*)'$/$1/;
	@utf8_alt_targets = $this->listify_comma_sep_string($utf8_alt_target_string_s);
	$numeric = $util->slot_value_in_double_colon_del_list($_, "num");
	$numeric =~ s/\s*$//;
	$annotation = $util->slot_value_in_double_colon_del_list($_, "annotation");
	$annotation =~ s/\s*$//;
	$lang_code = $util->slot_value_in_double_colon_del_list($_, "lcode");
	$prob = $util->slot_value_in_double_colon_del_list($_, "p") \|\| 1;
	unless (($utf8_target_string eq "") && ($numeric =~ /\d/)) {
	if ($lang_code) {
	$ht{UTF_CHAR_MAPPING_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = $prob;
	} else {
	$ht{UTF_CHAR_MAPPING}->{$utf8_source_string}->{$utf8_target_string} = $prob;
	}
	if ($word_external_punctuation_p) {
	if ($lang_code) {
	$ht{WORD_EXTERNAL_PUNCTUATION_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = $prob;
	} else {
	$ht{WORD_EXTERNAL_PUNCTUATION}->{$utf8_source_string}->{$utf8_target_string} = $prob;
	}
	}
	if ($this->braille_string_p($utf8_source_string)) {
	if (($utf8_target_string =~ /^[a-z]+$/)
	&& (! ($utf8_source_string =~ /^$braille_capital_letter_indicator/))) {
	my $uc_utf8_source_string = "$braille_capital_letter_indicator$utf8_source_string";
	my $uc_utf8_target_string = ucfirst $utf8_target_string;
	if ($lang_code) {
	$ht{UTF_CHAR_MAPPING_LANG_SPEC}->{$lang_code}->{$uc_utf8_source_string}->{$uc_utf8_target_string} = $prob;
	} else {
	$ht{UTF_CHAR_MAPPING}->{$uc_utf8_source_string}->{$uc_utf8_target_string} = $prob;
	}
	$this->register_word_boundary_info(*ht, $lang_code, $uc_utf8_source_string, $uc_utf8_target_string,
	$use_only_for_whole_word_p, $use_only_at_start_of_word_p, $use_only_at_end_of_word_p,
	$dont_use_at_start_of_word_p, $dont_use_at_end_of_word_p);
	}
	if (($utf8_target_string =~ /^[0-9]$/)
	&& ($utf8_source_string =~ /^$braille_number_indicator./)) {
	my $core_number_char = $utf8_source_string;
	$core_number_char =~ s/$braille_number_indicator//;
	$ht{BRAILLE_TO_DIGIT}->{$core_number_char} = $utf8_target_string;
	}
	}
	}
	if ($use_only_in_lower_case_enviroment_p) {
	if ($lang_code) {
	$ht{USE_ONLY_IN_LOWER_CASE_ENVIROMENT_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_target_string} = 1;
	} else {
	$ht{USE_ONLY_IN_LOWER_CASE_ENVIROMENT}->{$utf8_source_string}->{$utf8_target_string} = 1;
	}
	}
	$this->register_word_boundary_info(*ht, $lang_code, $utf8_source_string, $utf8_target_string,
	$use_only_for_whole_word_p, $use_only_at_start_of_word_p, $use_only_at_end_of_word_p,
	$dont_use_at_start_of_word_p, $dont_use_at_end_of_word_p);
	foreach $utf8_alt_target (@utf8_alt_targets) {
	if ($lang_code) {
	$ht{UTF_CHAR_ALT_MAPPING_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_alt_target} = $prob;
	$ht{USE_ALT_IN_POINTED_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_alt_target} = 1 if $use_alt_in_pointed_p;
	} else {
	$ht{UTF_CHAR_ALT_MAPPING}->{$utf8_source_string}->{$utf8_alt_target} = $prob;
	$ht{USE_ALT_IN_POINTED}->{$utf8_source_string}->{$utf8_alt_target} = 1 if $use_alt_in_pointed_p;
	}
	if ($use_only_for_whole_word_p) {
	if ($lang_code) {
	$ht{USE_ALT_ONLY_FOR_WHOLE_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_alt_target} = 1;
	} else {
	$ht{USE_ALT_ONLY_FOR_WHOLE_WORD}->{$utf8_source_string}->{$utf8_alt_target} = 1;
	}
	}
	if ($use_only_at_start_of_word_p) {
	if ($lang_code) {
	$ht{USE_ALT_ONLY_AT_START_OF_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_alt_target} = 1;
	} else {
	$ht{USE_ALT_ONLY_AT_START_OF_WORD}->{$utf8_source_string}->{$utf8_alt_target} = 1;
	}
	}
	if ($use_only_at_end_of_word_p) {
	if ($lang_code) {
	$ht{USE_ALT_ONLY_AT_END_OF_WORD_LANG_SPEC}->{$lang_code}->{$utf8_source_string}->{$utf8_alt_target} = 1;
	} else {
	$ht{USE_ALT_ONLY_AT_END_OF_WORD}->{$utf8_source_string}->{$utf8_alt_target} = 1;
	}
	}
	}
	if ($numeric =~ /\d/) {
	$ht{UTF_TO_NUMERIC}->{$utf8_source_string} = $numeric;
	}
	if ($annotation =~ /\S/) {
	$ht{UTF_ANNOTATION}->{$utf8_source_string} = $annotation;
	}
	$n++;
	}
	close(IN);
	# print STDERR "Loaded $n entries from $filename\n";
	} else {
	print STDERR "Can't open $filename\n";
	}
	}

	sub char_name_to_script {
	local($this, $char_name, *ht) = @_;

	return $cached_result if $cached_result = $ht{CHAR_NAME_TO_SCRIPT}->{$char_name};
	$orig_char_name = $char_name;
	$char_name =~ s/\s+(CONSONANT\|LETTER\|LIGATURE\|SIGN\|SYLLABLE\|SYLLABICS\|VOWEL)\b.*$//;
	my $script_name;
	while ($char_name) {
	last if $script_name = $ht{SCRIPT_NORM}->{(uc $char_name)};
	$char_name =~ s/\s\S+\s$//;
	}
	$script_name = "" unless defined($script_name);
	$ht{CHAR_NAME_TO_SCRIPT}->{$char_name} = $script_name;
	return $script_name;
	}

	sub letter_plus_char_p {
	local($this, $char_name) = @_;

	return $cached_result if $cached_result = $ht{CHAR_NAME_LETTER_PLUS}->{$char_name};
	my $letter_plus_p = ($char_name =~ /\b(?:LETTER\|VOWEL SIGN\|AU LENGTH MARK\|CONSONANT SIGN\|SIGN VIRAMA\|SIGN PAMAAEH\|SIGN COENG\|SIGN AL-LAKUNA\|SIGN ASAT\|SIGN ANUSVARA\|SIGN ANUSVARAYA\|SIGN BINDI\|TIPPI\|SIGN NIKAHIT\|SIGN CANDRABINDU\|SIGN VISARGA\|SIGN REAHMUK\|SIGN NUKTA\|SIGN DOT BELOW\|HEBREW POINT)\b/) ? 1 : 0;
	$ht{CHAR_NAME_LETTER_PLUS}->{$char_name} = $letter_plus_p;
	return $letter_plus_p;
	}

	sub subjoined_char_p {
	local($this, $char_name) = @_;

	return $cached_result if $cached_result = $ht{CHAR_NAME_SUBJOINED}->{$char_name};
	my $subjoined_p = (($char_name =~ /\b(?:SUBJOINED LETTER\|VOWEL SIGN\|AU LENGTH MARK\|EMPHASIS MARK\|CONSONANT SIGN\|SIGN VIRAMA\|SIGN PAMAAEH\|SIGN COENG\|SIGN ASAT\|SIGN ANUSVARA\|SIGN ANUSVARAYA\|SIGN BINDI\|TIPPI\|SIGN NIKAHIT\|SIGN CANDRABINDU\|SIGN VISARGA\|SIGN REAHMUK\|SIGN DOT BELOW\|HEBREW (POINT\|PUNCTUATION GERESH)\|ARABIC (?:DAMMA\|DAMMATAN\|FATHA\|FATHATAN\|HAMZA\|KASRA\|KASRATAN\|MADDAH\|SHADDA\|SUKUN))\b/)) ? 1 : 0;
	$ht{CHAR_NAME_SUBJOINED}->{$char_name} = $subjoined_p;
	return $subjoined_p;
	}

	sub new_node_id {
	local($this, *chart_ht) = @_;

	my $n_nodes = $chart_ht{N_NODES};
	$n_nodes++;
	$chart_ht{N_NODES} = $n_nodes;
	return $n_nodes;
	}

	sub add_node {
	local($this, $s, $start, $end, *chart_ht, $type, $comment) = @_;

	my $node_id = $this->new_node_id(*chart_ht);
	# print STDERR "add_node($node_id, $start-$end): $s [$comment]\n" if $comment =~ /number/;
	# print STDERR "add_node($node_id, $start-$end): $s [$comment]\n" if ($start >= 0) && ($start < 50);
	$chart_ht{NODE_START}->{$node_id} = $start;
	$chart_ht{NODE_END}->{$node_id} = $end;
	$chart_ht{NODES_STARTING_AT}->{$start}->{$node_id} = 1;
	$chart_ht{NODES_ENDING_AT}->{$end}->{$node_id} = 1;
	$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}->{$node_id} = 1;
	$chart_ht{NODE_TYPE}->{$node_id} = $type;
	$chart_ht{NODE_COMMENT}->{$node_id} = $comment;
	$chart_ht{NODE_ROMAN}->{$node_id} = $s;
	return $node_id;
	}

	sub get_node_for_span {
	local($this, $start, $end, *chart_ht) = @_;

	return "" unless defined($chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end});
	my @node_ids = sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}};

	return (@node_ids) ? $node_ids[0] : "";
	}

	sub get_node_for_span_and_type {
	local($this, $start, $end, *chart_ht, $type) = @_;

	return "" unless defined($chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end});
	my @node_ids = sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}};

	foreach $node_id (@node_ids) {
	return $node_id if $chart_ht{NODE_TYPE}->{$node_id} eq $type;
	}
	return "";
	}

	sub get_node_roman {
	local($this, $node_id, *chart_id, $default) = @_;

	$default = "" unless defined($default);
	my $roman = $chart_ht{NODE_ROMAN}->{$node_id};
	return (defined($roman)) ? $roman : $default;
	}

	sub set_node_id_slot_value {
	local($this, $node_id, $slot, $value, *chart_id) = @_;

	$chart_ht{NODE_SLOT}->{$node_id}->{$slot} = $value;
	}

	sub copy_slot_values {
	local($this, $old_node_id, $new_node_id, *chart_id, @slots) = @_;

	if (@slots) {
	foreach $slot (keys %{$chart_ht{NODE_SLOT}->{$old_node_id}}) {
	if (($slots[0] eq "all") \|\| $util->member($slot, @slots)) {
	my $value = $chart_ht{NODE_SLOT}->{$old_node_id}->{$slot};
	$chart_ht{NODE_SLOT}->{$new_node_id}->{$slot} = $value if defined($value);
	}
	}
	}
	}

	sub get_node_id_slot_value {
	local($this, $node_id, $slot, *chart_id, $default) = @_;

	$default = "" unless defined($default);
	my $value = $chart_ht{NODE_SLOT}->{$node_id}->{$slot};
	return (defined($value)) ? $value : $default;
	}

	sub get_node_for_span_with_slot_value {
	local($this, $start, $end, $slot, *chart_id, $default) = @_;

	$default = "" unless defined($default);
	return $default unless defined($chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end});
	my @node_ids = sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}};
	foreach $node_id (@node_ids) {
	my $value = $chart_ht{NODE_SLOT}->{$node_id}->{$slot};
	return $value if defined($value);
	}
	return $default;
	}

	sub get_node_for_span_with_slot {
	local($this, $start, $end, $slot, *chart_id, $default) = @_;

	$default = "" unless defined($default);
	return $default unless defined($chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end});
	my @node_ids = sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}};
	foreach $node_id (@node_ids) {
	my $value = $chart_ht{NODE_SLOT}->{$node_id}->{$slot};
	return $node_id if defined($value);
	}
	return $default;
	}

	sub register_new_complex_number_span_segment {
	local($this, $start, $mid, $end, *chart_id, $line_number) = @_;
	# e.g. 4 10 (= 40); 20 5 (= 25)
	# might become part of larger complex number span, e.g. 4 1000 3 100 20 1

	# print STDERR "register_new_complex_number_span_segment $start-$mid-$end\n" if $line_number == 43;
	if (defined($old_start = $chart_ht{COMPLEX_NUMERIC_END_START}->{$mid})) {
	undef($chart_ht{COMPLEX_NUMERIC_END_START}->{$mid});
	$chart_ht{COMPLEX_NUMERIC_START_END}->{$old_start} = $end;
	$chart_ht{COMPLEX_NUMERIC_END_START}->{$end} = $old_start;
	} else {
	$chart_ht{COMPLEX_NUMERIC_START_END}->{$start} = $end;
	$chart_ht{COMPLEX_NUMERIC_END_START}->{$end} = $start;
	}
	}

	sub romanize_by_token_with_caching {
	local($this, $s, $lang_code, $output_style, ht, pinyin_ht, $initial_char_offset, $control, $line_number) = @_;

	$control = "" unless defined($control);
	my $return_chart_p = ($control =~ /return chart/i);
	my $return_offset_mappings_p = ($control =~ /return offset mappings/i);
	return $this->romanize($s, $lang_code, $output_style, ht, pinyin_ht, $initial_char_offset, $control, $line_number)
	if $return_chart_p \|\| $return_offset_mappings_p;
	my $result = "";
	my @separators = ();
	my @tokens = ();
	$s =~ s/\n$//; # Added May 2, 2019 as bug-fix (duplicate empty lines)
	while (($sep, $token, $rest) = ($s =~ /^(\s)(\S+)(.)$/)) {
	push(@separators, $sep);
	push(@tokens, $token);
	$s = $rest;
	}
	push(@separators, $s);
	while (@tokens) {
	my $sep = shift @separators;
	my $token = shift @tokens;
	$result .= $sep;
	if ($token =~ /^[\x00-\x7F]*$/) { # all ASCII
	$result .= $token;
	} else {
	my $rom_token = $ht{CACHED_ROMANIZATION}->{$lang_code}->{$token};
	unless (defined($rom_token)) {
	$rom_token = $this->romanize($token, $lang_code, $output_style, ht, pinyin_ht, $initial_char_offset, $control, $line_number);
	$ht{CACHED_ROMANIZATION}->{$lang_code}->{$token} = $rom_token if defined($rom_token);
	}
	$result .= $rom_token;
	}
	}
	my $sep = shift @separators;
	$result .= $sep if defined($sep);

	return $result;
	}

	sub romanize {
	local($this, $s, $lang_code, $output_style, ht, pinyin_ht, $initial_char_offset, $control, $line_number, $initial_rom_char_offset) = @_;

	my $orig_lang_code = $lang_code;
	# Check whether the text (to be romanized) starts with a language code directive.
	if (($line_lang_code) = ($s =~ /^::lcode\s+([a-z][a-z][a-z])\s/)) {
	$lang_code = $line_lang_code;
	}
	$initial_char_offset = 0 unless defined($initial_char_offset);
	$initial_rom_char_offset = 0 unless defined($initial_rom_char_offset);
	$control = "" unless defined($control);
	my $return_chart_p = ($control =~ /return chart/i);
	my $return_offset_mappings_p = ($control =~ /return offset mappings/i);
	$line_number = "" unless defined($line_number);
	my @chars = $utf8->split_into_utf8_characters($s, "return only chars", *empty_ht);
	my $n_characters = $#chars + 1;
	%chart_ht = ();
	$chart_ht{N_CHARS} = $n_characters;
	$chart_ht{N_NODES} = 0;
	my $char = "";
	my $char_name = "";
	my $prev_script = "";
	my $current_script = "";
	my $script_start = 0;
	my $script_end = 0;
	my $prev_letter_plus_script = "";
	my $current_letter_plus_script = "";
	my $letter_plus_script_start = 0;
	my $letter_plus_script_end = 0;
	my $log ="";
	my $n_right_to_left_chars = 0;
	my $n_left_to_right_chars = 0;
	my $hebrew_word_start = ""; # used to identify Hebrew words with points
	my $hebrew_word_contains_point = 0;
	my $current_word_start = "";
	my $current_word_script = "";
	my $braille_all_caps_p = 0;

	# prep
	foreach $i ((0 .. ($#chars + 1))) {
	if ($i <= $#chars) {
	$char = $chars[$i];
	$chart_ht{ORIG_CHAR}->{$i} = $char;
	$char_name = $ht{UTF_TO_CHAR_NAME}->{$char} \|\| "";
	$chart_ht{CHAR_NAME}->{$i} = $char_name;
	$current_script = $this->char_name_to_script($char_name, *ht);
	$current_script_direction = $ht{DIRECTION}->{$current_script} \|\| '';
	if ($current_script_direction eq 'right-to-left') {
	$n_right_to_left_chars++;
	} elsif (($char =~ /^[a-z]$/i) \|\| ! ($char =~ /^[\x00-\x7F]$/)) {
	$n_left_to_right_chars++;
	}
	$chart_ht{CHAR_SCRIPT}->{$i} = $current_script;
	$chart_ht{SCRIPT_SEGMENT_START}->{$i} = ""; # default value, to be updated later
	$chart_ht{SCRIPT_SEGMENT_END}->{$i} = ""; # default value, to be updated later
	$chart_ht{LETTER_TOKEN_SEGMENT_START}->{$i} = ""; # default value, to be updated later
	$chart_ht{LETTER_TOKEN_SEGMENT_END}->{$i} = ""; # default value, to be updated later
	$subjoined_char_p = $this->subjoined_char_p($char_name);
	$chart_ht{CHAR_SUBJOINED}->{$i} = $subjoined_char_p;
	$letter_plus_char_p = $this->letter_plus_char_p($char_name);
	$chart_ht{CHAR_LETTER_PLUS}->{$i} = $letter_plus_char_p;
	$current_letter_plus_script = ($letter_plus_char_p) ? $current_script : "";
	$numeric_value = $ht{UTF_TO_NUMERIC}->{$char};
	$numeric_value = "" unless defined($numeric_value);
	$annotation = $ht{UTF_ANNOTATION}->{$char};
	$annotation = "" unless defined($annotation);
	$chart_ht{CHAR_NUMERIC_VALUE}->{$i} = $numeric_value;
	$chart_ht{CHAR_ANNOTATION}->{$i} = $annotation;
	$syllable_info = $ht{UTF_TO_SYLLABLE_INFO}->{$char} \|\| "";
	$chart_ht{CHAR_SYLLABLE_INFO}->{$i} = $syllable_info;
	$tone_mark = $ht{UTF_TO_TONE_MARK}->{$char} \|\| "";
	$chart_ht{CHAR_TONE_MARK}->{$i} = $tone_mark;
	} else {
	$char = "";
	$char_name = "";
	$current_script = "";
	$current_letter_plus_script = "";
	}
	if ($char_name =~ /^HEBREW (LETTER\|POINT\|PUNCTUATION GERESH) /) {
	$hebrew_word_start = $i if $hebrew_word_start eq "";
	$hebrew_word_contains_point = 1 if $char_name =~ /^HEBREW POINT /;
	} elsif ($hebrew_word_start ne "") {
	if ($hebrew_word_contains_point) {
	foreach $j (($hebrew_word_start .. ($i-1))) {
	$chart_ht{CHAR_PART_OF_POINTED_HEBREW_WORD}->{$j} = 1;
	}
	$chart_ht{CHAR_START_OF_WORD}->{$hebrew_word_start} = 1;
	$chart_ht{CHAR_END_OF_WORD}->{($i-1)} = 1;
	}
	$hebrew_word_start = "";
	$hebrew_word_contains_point = 0;
	}
	my $part_of_word_p = $current_script
	&& ($this->letter_plus_char_p($char_name)
	\|\| $this->subjoined_char_p($char_name)
	\|\| ($char_name =~ /\b(LETTER\|SYLLABLE\|SYLLABICS\|LIGATURE)\b/));

	# Braille punctuation
	my $end_offset = 0;
	if ($char_name =~ /^Braille\b/i) {
	if (($char =~ /^\s*$/) \|\| ($char_name =~ /BLANK/)) {
	$part_of_word_p = 0;
	$braille_all_caps_p = 0;
	} elsif ($chart_ht{NOT_PART_OF_WORD_P}->{$i}) {
	$part_of_word_p = 0;
	$braille_all_caps_p = 0;
	} elsif ((keys %{$ht{WORD_EXTERNAL_PUNCTUATION_LANG_SPEC}->{$lang_code}->{$char}})
	\|\| (keys %{$ht{WORD_EXTERNAL_PUNCTUATION}->{$char}})) {
	$part_of_word_p = 0;
	$braille_all_caps_p = 0;
	} elsif (($i+1 <= $#chars)
	&& ($s1 = $char . $chars[$i+1])
	&& ((keys %{$ht{WORD_EXTERNAL_PUNCTUATION_LANG_SPEC}->{$lang_code}->{$s1}})
	\|\| (keys %{$ht{WORD_EXTERNAL_PUNCTUATION}->{$s1}}))) {
	$part_of_word_p = 0;
	$braille_all_caps_p = 0;
	$chart_ht{NOT_PART_OF_WORD_P}->{($i+1)} = 1;
	} elsif (($i+2 <= $#chars)
	&& ($s2 = $char . $chars[$i+1] . $chars[$i+2])
	&& ((keys %{$ht{WORD_EXTERNAL_PUNCTUATION_LANG_SPEC}->{$lang_code}->{$s2}})
	\|\| (keys %{$ht{WORD_EXTERNAL_PUNCTUATION}->{$s2}}))) {
	$part_of_word_p = 0;
	$braille_all_caps_p = 0;
	$chart_ht{NOT_PART_OF_WORD_P}->{($i+1)} = 1;
	$chart_ht{NOT_PART_OF_WORD_P}->{($i+2)} = 1;
	} elsif (($i+1 <= $#chars)
	&& ($char eq $braille_capital_letter_indicator)
	&& ($chars[$i+1] eq $braille_capital_letter_indicator)) {
	$braille_all_caps_p = 1;
	} else {
	$part_of_word_p = 1;
	}
	# last period in Braille text is also not part_of_word_p
	if (($char eq $braille_period)
	&& (($i == $#chars)
	\|\| (($i < $#chars)
	&& (! $this->braille_string_p($chars[$i+1]))))) {
	$part_of_word_p = 0;
	}
	# period before other word-external punctuation is also not part_of_word_p
	if (($i > 0)
	&& ($chars[$i-1] eq $braille_period)
	&& (! $part_of_word_p)
	&& ($current_word_start ne "")) {
	$end_offset = -1;
	}
	} else {
	$braille_all_caps_p = 0;
	}
	$chart_ht{BRAILLE_ALL_CAPS_P}->{$i} = $braille_all_caps_p;

	if (($current_word_start ne "")
	&& ((! $part_of_word_p)
	\|\| ($current_script ne $current_word_script))) {
	# END OF WORD
	$chart_ht{CHAR_START_OF_WORD}->{$current_word_start} = 1;
	$chart_ht{CHAR_END_OF_WORD}->{($i-1+$end_offset)} = 1;
	my $word = join("", @chars[$current_word_start .. ($i-1+$end_offset)]);
	$chart_ht{WORD_START_END}->{$current_word_start}->{$i} = $word;
	$chart_ht{WORD_END_START}->{$i+$end_offset}->{$current_word_start} = $word;
	# print STDERR "Word ($current_word_start-$i+$end_offset): $word ($current_word_script)\n";
	$current_word_start = "";
	$current_word_script = "";
	}
	if ($part_of_word_p && ($current_word_start eq "")) {
	# START OF WORD
	$current_word_start = $i;
	$current_word_script = $current_script;
	}
	# print STDERR "$i char: $char ($current_script)\n";
	unless ($current_script eq $prev_script) {
	if ($prev_script && ($i-1 >= $script_start)) {
	my $script_end = $i;
	$chart_ht{SCRIPT_SEGMENT_START_TO_END}->{$script_start} = $script_end;
	$chart_ht{SCRIPT_SEGMENT_END_TO_START}->{$script_end} = $script_start;
	foreach $i (($script_start .. $script_end)) {
	$chart_ht{SCRIPT_SEGMENT_START}->{$i} = $script_start;
	$chart_ht{SCRIPT_SEGMENT_END}->{$i} = $script_end;
	}
	# print STDERR "Script segment $script_start-$script_end: $prev_script\n";
	}
	$script_start = $i;
	}
	unless ($current_letter_plus_script eq $prev_letter_plus_script) {
	if ($prev_letter_plus_script && ($i-1 >= $letter_plus_script_start)) {
	my $letter_plus_script_end = $i;
	$chart_ht{LETTER_TOKEN_SEGMENT_START_TO_END}->{$letter_plus_script_start} = $letter_plus_script_end;
	$chart_ht{LETTER_TOKEN_SEGMENT_END_TO_START}->{$letter_plus_script_end} = $letter_plus_script_start;
	foreach $i (($letter_plus_script_start .. $letter_plus_script_end)) {
	$chart_ht{LETTER_TOKEN_SEGMENT_START}->{$i} = $letter_plus_script_start;
	$chart_ht{LETTER_TOKEN_SEGMENT_END}->{$i} = $letter_plus_script_end;
	}
	# print STDERR "Script token segment $letter_plus_script_start-$letter_plus_script_end: $prev_letter_plus_script\n";
	}
	$letter_plus_script_start = $i;
	}
	$prev_script = $current_script;
	$prev_letter_plus_script = $current_letter_plus_script;
	}
	$ht{STRING_IS_DOMINANTLY_RIGHT_TO_LEFT}->{$s} = 1 if $n_right_to_left_chars > $n_left_to_right_chars;

	# main
	my $i = 0;
	while ($i <= $#chars) {
	my $char = $chart_ht{ORIG_CHAR}->{$i};
	my $current_script = $chart_ht{CHAR_SCRIPT}->{$i};
	$chart_ht{CHART_CONTAINS_SCRIPT}->{$current_script} = 1;
	my $script_segment_start = $chart_ht{SCRIPT_SEGMENT_START}->{$i};
	my $script_segment_end = $chart_ht{SCRIPT_SEGMENT_END}->{$i};
	my $char_name = $chart_ht{CHAR_NAME}->{$i};
	my $subjoined_char_p = $chart_ht{CHAR_SUBJOINED}->{$i};
	my $letter_plus_char_p = $chart_ht{CHAR_LETTER_PLUS}->{$i};
	my $numeric_value = $chart_ht{CHAR_NUMERIC_VALUE}->{$i};
	my $annotation = $chart_ht{CHAR_ANNOTATION}->{$i};
	# print STDERR " $char_name annotation: $annotation\n" if $annotation;
	my $tone_mark = $chart_ht{CHAR_TONE_MARK}->{$i};
	my $found_char_mapping_p = 0;
	my $prev_char_name = ($i >= 1) ? $chart_ht{CHAR_NAME}->{($i-1)} : "";
	my $prev2_script = ($i >= 2) ? $chart_ht{CHAR_SCRIPT}->{($i-2)} : "";
	my $prev_script = ($i >= 1) ? $chart_ht{CHAR_SCRIPT}->{($i-1)} : "";
	my $next_script = ($i < $#chars) ? $chart_ht{CHAR_SCRIPT}->{($i+1)} : "";
	my $next_char = ($i < $#chars) ? $chart_ht{ORIG_CHAR}->{($i+1)} : "";
	my $next_char_name = $ht{UTF_TO_CHAR_NAME}->{$next_char} \|\| "";
	my $prev2_letter_plus_char_p = ($i >= 2) ? $chart_ht{CHAR_LETTER_PLUS}->{($i-2)} : 0;
	my $prev_letter_plus_char_p = ($i >= 1) ? $chart_ht{CHAR_LETTER_PLUS}->{($i-1)} : 0;
	my $next_letter_plus_char_p = ($i < $#chars) ? $chart_ht{CHAR_LETTER_PLUS}->{($i+1)} : 0;
	my $next_index = $i + 1;

	# Braille numeric mode
	if ($char eq $braille_number_indicator) {
	my $offset = 0;
	my $numeric_value = "";
	my $digit;
	while ($i+$offset < $#chars) {
	$offset++;
	my $offset_char = $chart_ht{ORIG_CHAR}->{$i+$offset};
	if (defined($digit = $ht{BRAILLE_TO_DIGIT}->{$offset_char})) {
	$numeric_value .= $digit;
	} elsif (($offset_char eq $braille_decimal_point)
	\|\| ($ht{UTF_CHAR_MAPPING}->{$offset_char}->{"."})) {
	$numeric_value .= ".";
	} elsif ($offset_char eq $braille_comma) {
	$numeric_value .= ",";
	} elsif ($offset_char eq $braille_numeric_space) {
	$numeric_value .= " ";
	} elsif ($offset_char eq $braille_solidus) {
	$numeric_value .= "/";
	} elsif ($offset_char eq $braille_number_indicator) {
	# stay in Braille numeric mode
	} elsif ($offset_char eq $braille_letter_indicator) {
	# consider as part of number, but without contributing to numeric_value
	last;
	} else {
	$offset--;
	last;
	}
	}
	if ($offset) {
	$next_index = $i + $offset + 1;
	$node_id = $this->add_node($numeric_value, $i, $next_index, *chart_ht, "", "braille number");
	$found_char_mapping_p = 1;
	}
	}

	unless ($found_char_mapping_p) {
	foreach $string_length (reverse(1 .. 6)) {
	next if ($i + $string_length-1) > $#chars;
	my $start_of_word_p = $chart_ht{CHAR_START_OF_WORD}->{$i} \|\| 0;
	my $end_of_word_p = $chart_ht{CHAR_END_OF_WORD}->{($i+$string_length-1)} \|\| 0;
	my $multi_char_substring = join("", @chars[$i..($i+$string_length-1)]);
	my @mappings = keys %{$ht{UTF_CHAR_MAPPING_LANG_SPEC}->{$lang_code}->{$multi_char_substring}};
	@mappings = keys %{$ht{UTF_CHAR_MAPPING}->{$multi_char_substring}} unless @mappings;
	my @mappings_whole = ();
	my @mappings_start_or_end = ();
	my @mappings_other = ();
	foreach $mapping (@mappings) {
	next if $mapping =~ /$__.*__$/;
	if ($ht{USE_ONLY_FOR_WHOLE_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$mapping}
	\|\| $ht{USE_ONLY_FOR_WHOLE_WORD}->{$multi_char_substring}->{$mapping}) {
	push(@mappings_whole, $mapping) if $start_of_word_p && $end_of_word_p;
	} elsif ($ht{USE_ONLY_AT_START_OF_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$mapping}
	\|\| $ht{USE_ONLY_AT_START_OF_WORD}->{$multi_char_substring}->{$mapping}) {
	push(@mappings_start_or_end, $mapping) if $start_of_word_p;
	} elsif ($ht{USE_ONLY_AT_END_OF_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$mapping}
	\|\| $ht{USE_ONLY_AT_END_OF_WORD}->{$multi_char_substring}->{$mapping}) {
	push(@mappings_start_or_end, $mapping) if $end_of_word_p;
	} else {
	push(@mappings_other, $mapping);
	}
	}
	@mappings = @mappings_whole;
	@mappings = @mappings_start_or_end unless @mappings;
	@mappings = @mappings_other unless @mappings;
	foreach $mapping (@mappings) {
	next if $mapping =~ /$__.*__$/;
	if ($ht{DONT_USE_AT_START_OF_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$mapping}
	\|\| $ht{DONT_USE_AT_START_OF_WORD}->{$multi_char_substring}->{$mapping}) {
	next if $start_of_word_p;
	}
	if ($ht{DONT_USE_AT_END_OF_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$mapping}
	\|\| $ht{DONT_USE_AT_END_OF_WORD}->{$multi_char_substring}->{$mapping}) {
	next if $end_of_word_p;
	}
	my $mapping2 = ($chart_ht{BRAILLE_ALL_CAPS_P}->{$i}) ? (uc $mapping) : $mapping;
	$node_id = $this->add_node($mapping2, $i, $i+$string_length, *chart_ht, "", "multi-char-mapping");
	$next_index = $i + $string_length;
	$found_char_mapping_p = 1;
	if ($annotation) {
	@annotation_elems = split(/,\s*/, $annotation);
	foreach $annotation_elem (@annotation_elems) {
	if (($a_slot, $a_value) = ($annotation_elem =~ /^(\S+?):(\S+)\s*$/)) {
	$this->set_node_id_slot_value($node_id, $a_slot, $a_value, *chart_ht);
	} else {
	$this->set_node_id_slot_value($node_id, $annotation_elem, 1, *chart_ht);
	}
	}
	}
	}
	my @alt_mappings = keys %{$ht{UTF_CHAR_ALT_MAPPING_LANG_SPEC}->{$lang_code}->{$multi_char_substring}};
	@alt_mappings = keys %{$ht{UTF_CHAR_ALT_MAPPING}->{$multi_char_substring}} unless @alt_mappings;
	@alt_mappings = () if ($#alt_mappings == 0) && ($alt_mappings[0] eq "_NONE_");
	foreach $alt_mapping (@alt_mappings) {
	if ($chart_ht{CHAR_PART_OF_POINTED_HEBREW_WORD}->{$i}) {
	next unless
	$ht{USE_ALT_IN_POINTED_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$alt_mapping}
	\|\| $ht{USE_ALT_IN_POINTED}->{$multi_char_substring}->{$alt_mapping};
	}
	if ($ht{USE_ALT_ONLY_FOR_WHOLE_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$alt_mapping}
	\|\| $ht{USE_ALT_ONLY_FOR_WHOLE_WORD}->{$multi_char_substring}->{$alt_mapping}) {
	next unless $start_of_word_p && $end_of_word_p;
	}
	if ($ht{USE_ALT_ONLY_AT_START_OF_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$alt_mapping}
	\|\| $ht{USE_ALT_ONLY_AT_START_OF_WORD}->{$multi_char_substring}->{$alt_mapping}) {
	next unless $start_of_word_p;
	}
	if ($ht{USE_ALT_ONLY_AT_END_OF_WORD_LANG_SPEC}->{$lang_code}->{$multi_char_substring}->{$alt_mapping}
	\|\| $ht{USE_ALT_ONLY_AT_END_OF_WORD}->{$multi_char_substring}->{$alt_mapping}) {
	next unless $end_of_word_p;
	}
	my $alt_mapping2 = ($chart_ht{BRAILLE_ALL_CAPS_P}->{$i}) ? (uc $alt_mapping) : $alt_mapping;
	$node_id = $this->add_node($alt_mapping2, $i, $i+$string_length, *chart_ht, "alt", "multi-char-mapping");
	if ($annotation) {
	@annotation_elems = split(/,\s*/, $annotation);
	foreach $annotation_elem (@annotation_elems) {
	if (($a_slot, $a_value) = ($annotation_elem =~ /^(\S+?):(\S+)\s*$/)) {
	$this->set_node_id_slot_value($node_id, $a_slot, $a_value, *chart_ht);
	} else {
	$this->set_node_id_slot_value($node_id, $annotation_elem, 1, *chart_ht);
	}
	}
	}
	}
	}
	}
	unless ($found_char_mapping_p) {
	my $prev_node_id = $this->get_node_for_span($i-4, $i, *chart_ht)
	\|\| $this->get_node_for_span($i-3, $i, *chart_ht)
	\|\| $this->get_node_for_span($i-2, $i, *chart_ht)
	\|\| $this->get_node_for_span($i-1, $i, *chart_ht);
	my $prev_char_roman = ($prev_node_id) ? $this->get_node_roman($prev_node_id, *chart_id) : "";
	my $prev_node_start = ($prev_node_id) ? $chart_ht{NODE_START}->{$prev_node_id} : "";

	# Number
	if (($numeric_value =~ /\d/)
	&& (! ($char_name =~ /SUPERSCRIPT/))) {
	my $prev_numeric_value = $this->get_node_for_span_with_slot_value($i-1, $i, "numeric-value", *chart_id);
	my $sep = "";
	$sep = " " if ($char_name =~ /^vulgar fraction /i) && ($prev_numeric_value =~ /\d/);
	$node_id = $this->add_node("$sep$numeric_value", $i, $i+1, *chart_ht, "", "number");
	$this->set_node_id_slot_value($node_id, "numeric-value", $numeric_value, *chart_ht);
	if ((($prev_numeric_value =~ /\d/) && ($numeric_value =~ /\d\d/))
	\|\| (($prev_numeric_value =~ /\d\d/) && ($numeric_value =~ /\d/))) {
	# pull in any other parts of single digits
	my $j = 1;
	# pull in any single digits adjoining on left
	if ($prev_numeric_value =~ /^\d$/) {
	while (1) {
	if (($i-$j-1 >= 0)
	&& defined($digit_value = $this->get_node_for_span_with_slot_value($i-$j-1, $i-$j, "numeric-value", *chart_id))
	&& ($digit_value =~ /^\d$/)) {
	$j++;
	} elsif (($i-$j-2 >= 0)
	&& ($chart_ht{ORIG_CHAR}->{($i-$j-1)} =~ /^[.,]$/)
	&& defined($digit_value = $this->get_node_for_span_with_slot_value($i-$j-2, $i-$j-1, "numeric-value", *chart_id))
	&& ($digit_value =~ /^\d$/)) {
	$j += 2;
	} else {
	last;
	}
	}
	}
	# pull in any single digits adjoining on right
	my $k = 0;
	if ($numeric_value =~ /^\d$/) {
	while (1) {
	if (defined($next_numeric_value = $chart_ht{CHAR_NUMERIC_VALUE}->{($i+$k+1)})
	&& ($next_numeric_value =~ /^\d$/)) {
	$k++;
	} else {
	last;
	}
	}
	}
	$this->register_new_complex_number_span_segment($i-$j, $i, $i+$k+1, *chart_ht, $line_number);
	}
	if ($chinesePM->string_contains_utf8_cjk_unified_ideograph_p($char)
	&& ($tonal_translit = $chinesePM->tonal_pinyin($char, *pinyin_ht, ""))) {
	$de_accented_translit = $util->de_accent_string($tonal_translit);
	if ($numeric_value =~ /^(10000\|1000000000000\|10000000000000000)$/) {
	$chart_ht{NODE_TYPE}->{$node_id} = "alt"; # keep, but demote
	$alt_node_id = $this->add_node($de_accented_translit, $i, $i+1, *chart_ht, "", "CJK");
	} else {
	$alt_node_id = $this->add_node($de_accented_translit, $i, $i+1, *chart_ht, "alt", "CJK");
	}
	}

	# ASCII
	} elsif ($char =~ /^[\x00-\x7F]$/) {
	$this->add_node($char, $i, $i+1, *chart_ht, "", "ASCII"); # ASCII character, incl. control characters

	# Emoji, dingbats, pictographs
	} elsif ($char =~ /^(\xE2[\x98-\x9E]\|\xF0\x9F[\x8C-\xA7])/) {
	$this->add_node($char, $i, $i+1, *chart_ht, "", "pictograph");

	# Hangul (Korean)
	} elsif (($char =~ /^[\xEA-\xED]/)
	&& ($romanized_char = $this->unicode_hangul_romanization($char))) {
	$this->add_node($romanized_char, $i, $i+1, *chart_ht, "", "Hangul");

	# CJK (Chinese, Japanese, Korean)
	} elsif ($chinesePM->string_contains_utf8_cjk_unified_ideograph_p($char)
	&& ($tonal_translit = $chinesePM->tonal_pinyin($char, *pinyin_ht, ""))) {
	$de_accented_translit = $util->de_accent_string($tonal_translit);
	$this->add_node($de_accented_translit, $i, $i+1, *chart_ht, "", "CJK");

	# Virama (cancel preceding vowel in Abudiga scripts)
	} elsif ($char_name =~ /\bSIGN (?:VIRAMA\|AL-LAKUNA\|ASAT\|COENG\|PAMAAEH)\b/) {
	# VIRAMA: cancel preceding default vowel (in Abudiga scripts)
	if (($prev_script eq $current_script)
	&& (($prev_char_roman_consonant, $prev_char_roman_vowel) = ($prev_char_roman =~ /^(.*[bcdfghjklmnpqrstvwxyz])([aeiou]+)$/i))
	&& ($ht{SCRIPT_ABUDIGA_DEFAULT_VOWEL}->{$current_script}->{(lc $prev_char_roman_vowel)})) {
	$this->add_node($prev_char_roman_consonant, $prev_node_start, $i+1, *chart_ht, "", "virama");
	} else {
	$this->add_node("", $i, $i+1, *chart_ht, "", "unexpected-virama");
	}

	# Nukta (special (typically foreign) variant)
	} elsif ($char_name =~ /\bSIGN (?:NUKTA)\b/) {
	# NUKTA (dot): indicates special (typically foreign) variant; normally covered by multi-mappings
	if ($prev_script eq $current_script) {
	my $node_id = $this->add_node($prev_char_roman, $prev_node_start, $i+1, *chart_ht, "", "nukta");
	$this->copy_slot_values($prev_node_id, $node_id, *chart_id, "all");
	$this->set_node_id_slot_value($node_id, "nukta", 1, *chart_ht);
	} else {
	$this->add_node("", $i, $i+1, *chart_ht, "", "unexpected-nukta");
	}

	# Zero-width character, incl. zero width space/non-joiner/joiner, left-to-right/right-to-left mark
	} elsif ($char =~ /^\xE2\x80[\x8B-\x8F\xAA-\xAE]$/) {
	if ($prev_node_id) {
	my $node_id = $this->add_node($prev_char_roman, $prev_node_start, $i+1, *chart_ht, "", "zero-width-char");
	$this->copy_slot_values($prev_node_id, $node_id, *chart_id, "all");
	} else {
	$this->add_node("", $i, $i+1, *chart_ht, "", "zero-width-char");
	}
	} elsif (($char =~ /^\xEF\xBB\xBF$/) && $prev_node_id) { # OK to leave byte-order-mark at beginning of line
	my $node_id = $this->add_node($prev_char_roman, $prev_node_start, $i+1, *chart_ht, "", "zero-width-char");
	$this->copy_slot_values($prev_node_id, $node_id, *chart_id, "all");

	# Tone mark
	} elsif ($tone_mark) {
	if ($prev_script eq $current_script) {
	my $node_id = $this->add_node($prev_char_roman, $prev_node_start, $i+1, *chart_ht, "", "tone-mark");
	$this->copy_slot_values($prev_node_id, $node_id, *chart_id, "all");
	$this->set_node_id_slot_value($node_id, "tone-mark", $tone_mark, *chart_ht);
	} else {
	$this->add_node("", $i, $i+1, *chart_ht, "", "unexpected-tone-mark");
	}

	# Diacritic
	} elsif (($char_name =~ /\b(ACCENT\|TONE\|COMBINING DIAERESIS\|COMBINING DIAERESIS BELOW\|COMBINING MACRON\|COMBINING VERTICAL LINE ABOVE\|COMBINING DOT ABOVE RIGHT\|COMBINING TILDE\|COMBINING CYRILLIC\|MUUSIKATOAN\|TRIISAP)\b/) && ($ht{UTF_TO_CAT}->{$char} =~ /^Mn/)) {
	if ($prev_script eq $current_script) {
	my $node_id = $this->add_node($prev_char_roman, $prev_node_start, $i+1, *chart_ht, "", "diacritic");
	$this->copy_slot_values($prev_node_id, $node_id, *chart_id, "all");
	$diacritic = lc $char_name;
	$diacritic =~ s/^.*(?:COMBINING CYRILLIC\|COMBINING\|SIGN)\s+//i;
	$diacritic =~ s/^.*(ACCENT\|TONE)/$1/i;
	$diacritic =~ s/^\s*//;
	$this->set_node_id_slot_value($node_id, "diacritic", $diacritic, *chart_ht);
	# print STDERR "diacritic: $diacritic\n";
	} else {
	$this->add_node("", $i, $i+1, *chart_ht, "", "unexpected-diacritic");
	}

	# Romanize to find out more
	} elsif ($char_name) {
	if (defined($romanized_char = $this->romanize_char_at_position($i, $lang_code, $output_style, ht, chart_ht))) {
	# print STDERR "ROM l.$line_number/$i: $romanized_char\n" if $line_number =~ /^[12]$/;
	print STDOUT "ROM l.$line_number/$i: $romanized_char\n" if $verbosePM;

	# Empty string mapping
	if ($romanized_char eq "\"\"") {
	$this->add_node("", $i, $i+1, *chart_ht, "", "empty-string-mapping");
	# consider adding something for implausible romanizations of length 6+

	# keep original character (instead of romanized_char lengthener, character-18b00 etc.)
	} elsif (($romanized_char =~ /^(character\|lengthener\|modifier)/)) {
	$this->add_node($char, $i, $i+1, *chart_ht, "", "nevermind-keep-original");

	# Syllabic suffix in Abudiga languages, e.g. -m, -ng
	} elsif (($romanized_char =~ /^\+(H\|M\|N\|NG)$/i)
	&& ($prev_script eq $current_script)
	&& ($ht{SCRIPT_ABUDIGA_DEFAULT_VOWEL}->{$current_script}->{"a"})) {
	my $core_suffix = $romanized_char;
	$core_suffix =~ s/^\+//;
	if ($prev_char_roman =~ /[aeiou]$/i) {
	$this->add_node($core_suffix, $i, $i+1, *chart_ht, "", "syllable-end-consonant");
	} else {
	$this->add_node(join("", $prev_char_roman, "a", $core_suffix), $prev_node_start, $i+1, *chart_ht, "", "syllable-end-consonant-with-added-a");
	$this->add_node(join("", "a", $core_suffix), $i, $i+1, *chart_ht, "backup", "syllable-end-consonant");
	}

	# Japanese special cases
	} elsif ($char_name =~ /(?:HIRAGANA\|KATAKANA) LETTER SMALL Y/) {
	if (($prev_script eq $current_script)
	&& (($prev_char_roman_consonant) = ($prev_char_roman =~ /^(.*[bcdfghjklmnpqrstvwxyz])i$/i))) {
	unless ($this->get_node_for_span_and_type($prev_node_start, $i+1, *chart_ht, "")) {
	$this->add_node("$prev_char_roman_consonant$romanized_char", $prev_node_start, $i+1, *chart_ht, "", "japanese-contraction");
	}
	} else {
	$this->add_node($romanized_char, $i, $i+1, *chart_ht, "", "unexpected-japanese-contraction-character");
	}
	} elsif (($prev_script =~ /^(HIRAGANA\|KATAKANA)$/i)
	&& ($char_name eq "KATAKANA-HIRAGANA PROLONGED SOUND MARK") # Choonpu
	&& (($prev_char_roman_vowel) = ($prev_char_roman =~ /([aeiou])$/i))) {
	$this->add_node("$prev_char_roman$prev_char_roman_vowel", $prev_node_start, $i+1, *chart_ht, "", "japanese-vowel-lengthening");
	} elsif (($current_script =~ /^(Hiragana\|Katakana)$/i)
	&& ($char_name =~ /^(HIRAGANA\|KATAKANA) LETTER SMALL TU$/i) # Sokuon/Sukun
	&& ($next_script eq $current_script)
	&& ($romanized_next_char = $this->romanize_char_at_position_incl_multi($i+1, $lang_code, $output_style, ht, chart_ht))
	&& (($doubled_consonant) = ($romanized_next_char =~ /^(ch\|[bcdfghjklmnpqrstwz])/i))) {
	# Note: $romanized_next_char could be part of a multi-character mapping
	# print STDERR "current_script: $current_script char_name: $char_name next_script: $next_script romanized_next_char: $romanized_next_char doubled_consonant: $doubled_consonant\n";
	$doubled_consonant = "t" if $doubled_consonant eq "ch";
	$this->add_node($doubled_consonant, $i, $i+1, *chart_ht, "", "japanese-consonant-doubling");

	# Greek small letter mu to micro-sign (instead of to "m") as used in abbreviations for microgram/micrometer/microliter/microsecond/micromolar/microfarad etc.
	} elsif (($char_name eq "GREEK SMALL LETTER MU")
	&& (! ($prev_script =~ /^GREEK$/))
	&& ($i < $#chars)
	&& ($chart_ht{ORIG_CHAR}->{($i+1)} =~ /^[cfgjlmstv]$/i)) {
	$this->add_node("\xC2\xB5", $i, $i+1, *chart_ht, "", "greek-mu-to-micro-sign");

	# Gurmukhi addak (doubles following consonant)
	} elsif (($current_script eq "Gurmukhi")
	&& ($char_name eq "GURMUKHI ADDAK")) {
	if (($next_script eq $current_script)
	&& ($romanized_next_char = $this->romanize_char_at_position_incl_multi($i+1, $lang_code, $output_style, ht, chart_ht))
	&& (($doubled_consonant) = ($romanized_next_char =~ /^([bcdfghjklmnpqrstvwxz])/i))) {
	$this->add_node($doubled_consonant, $i, $i+1, *chart_ht, "", "gurmukhi-consonant-doubling");
	} else {
	$this->add_node("'", $i, $i+1, *chart_ht, "", "gurmukhi-unexpected-addak");
	}

	# Subjoined character
	} elsif ($subjoined_char_p
	&& ($prev_script eq $current_script)
	&& (($prev_char_roman_consonant, $prev_char_roman_vowel) = ($prev_char_roman =~ /^(.*[bcdfghjklmnpqrstvwxyz])([aeiou]+)$/i))
	&& ($ht{SCRIPT_ABUDIGA_DEFAULT_VOWEL}->{$current_script}->{(lc $prev_char_roman_vowel)})) {
	my $new_roman = "$prev_char_roman_consonant$romanized_char";
	$this->add_node($new_roman, $prev_node_start, $i+1, *chart_ht, "", "subjoined-character");
	# print STDERR " Subjoin l.$line_number/$i: $new_roman\n" if $line_number =~ /^[12]$/;

	# Thai special case: written-pre-consonant-spoken-post-consonant
	} elsif (($char_name =~ /THAI CHARACTER/)
	&& ($prev_script eq $current_script)
	&& ($chart_ht{CHAR_SYLLABLE_INFO}->{($i-1)} =~ /written-pre-consonant-spoken-post-consonant/i)
	&& ($prev_char_roman =~ /^[aeiou]+$/i)
	&& ($romanized_char =~ /^[bcdfghjklmnpqrstvwxyz]/)) {
	$this->add_node("$romanized_char$prev_char_roman", $prev_node_start, $i+1, *chart_ht, "", "thai-vowel-consonant-swap");

	# Thai special case: THAI CHARACTER O ANG (U+0E2D "\xE0\xB8\xAD")
	} elsif ($char_name eq "THAI CHARACTER O ANG") {
	if ($prev_script ne $current_script) {
	$this->add_node("", $i, $i+1, *chart_ht, "", "thai-initial-o-ang-drop");
	} elsif ($next_script ne $current_script) {
	$this->add_node("", $i, $i+1, *chart_ht, "", "thai-final-o-ang-drop");
	} else {
	my $romanized_next_char = $this->romanize_char_at_position($i+1, $lang_code, $output_style, ht, chart_ht);
	my $romanized_prev2_char = $this->romanize_char_at_position($i-2, $lang_code, $output_style, ht, chart_ht);
	if (($prev_char_roman =~ /^[bcdfghjklmnpqrstvwxz]+$/i)
	&& ($romanized_next_char =~ /^[bcdfghjklmnpqrstvwxz]+$/i)) {
	$this->add_node("o", $i, $i+1, *chart_ht, "", "thai-middle-o-ang"); # keep between consonants
	} elsif (($prev2_script eq $current_script)
	&& 0
	&& ($prev_char_name =~ /^THAI CHARACTER MAI [A-Z]+$/) # Thai tone
	&& ($romanized_prev2_char =~ /^[bcdfghjklmnpqrstvwxz]+$/i)
	&& ($romanized_next_char =~ /^[bcdfghjklmnpqrstvwxz]+$/i)) {
	$this->add_node("o", $i, $i+1, *chart_ht, "", "thai-middle-o-ang"); # keep between consonant+tone-mark and consonant
	} else {
	$this->add_node("", $i, $i+1, *chart_ht, "", "thai-middle-o-ang-drop"); # drop next to vowel
	}
	}

	# Romanization with space
	} elsif ($romanized_char =~ /\s/) {
	$this->add_node($char, $i, $i+1, *chart_ht, "", "space");

	# Tibetan special cases
	} elsif ($current_script eq "Tibetan") {

	if ($subjoined_char_p
	&& ($prev_script eq $current_script)
	&& $prev_letter_plus_char_p
	&& ($prev_char_roman =~ /^[bcdfghjklmnpqrstvwxyz]+$/i)) {
	$this->add_node("$prev_char_roman$romanized_char", $prev_node_start, $i+1, *chart_ht, "", "subjoined-tibetan-character");
	} elsif ($romanized_char =~ /^-A$/i) {
	my $romanized_next_char = $this->romanize_char_at_position($i+1, $lang_code, $output_style, ht, chart_ht);
	if (! $prev_letter_plus_char_p) {
	$this->add_node("'", $i, $i+1, *chart_ht, "", "tibetan-frontal-dash-a");
	} elsif (($prev_script eq $current_script)
	&& ($next_script eq $current_script)
	&& ($prev_char_roman =~ /[bcdfghjklmnpqrstvwxyz]$/)
	&& ($romanized_next_char =~ /^[aeiou]/)) {
	$this->add_node("a'", $i, $i+1, *chart_ht, "", "tibetan-medial-dash-a");
	} elsif (($prev_script eq $current_script)
	&& ($next_script eq $current_script)
	&& ($prev_char_roman =~ /[aeiou]$/)
	&& ($romanized_next_char =~ /[aeiou]/)) {
	$this->add_node("'", $i, $i+1, *chart_ht, "", "tibetan-reduced-medial-dash-a");
	} elsif (($prev_script eq $current_script)
	&& (! ($prev_char_roman =~ /[aeiou]/))
	&& (! $next_letter_plus_char_p)) {
	$this->add_node("a", $i, $i+1, *chart_ht, "", "tibetan-final-dash-a");
	} else {
	$this->add_node("a", $i, $i+1, *chart_ht, "", "unexpected-tibetan-dash-a");
	}
	} elsif (($romanized_char =~ /^[AEIOU]/i)
	&& ($prev_script eq $current_script)
	&& ($prev_char_roman =~ /^A$/i)
	&& (! $prev2_letter_plus_char_p)) {
	$this->add_node($romanized_char, $prev_node_start, $i+1, *chart_ht, "", "tibetan-dropped-word-initial-a");
	} else {
	$this->add_node($romanized_char, $i, $i+1, *chart_ht, "", "standard-unicode-based-romanization");
	}

	# Khmer (for MUUSIKATOAN etc. see under "Diacritic" above)
	} elsif (($current_script eq "Khmer")
	&& (($char_roman_consonant, $char_roman_vowel) = ($romanized_char =~ /^(.*[bcdfghjklmnpqrstvwxyz])([ao]+)-$/i))) {
	my $romanized_next_char = $this->romanize_char_at_position($i+1, $lang_code, $output_style, ht, chart_ht);
	if (($next_script eq $current_script)
	&& ($romanized_next_char =~ /^[aeiouy]/i)) {
	$this->add_node($char_roman_consonant, $i, $i+1, *chart_ht, "", "khmer-vowel-drop");
	} else {
	$this->add_node("$char_roman_consonant$char_roman_vowel", $i, $i+1, *chart_ht, "", "khmer-standard-unicode-based-romanization");
	}

	# Abudiga add default vowel
	} elsif ((@abudiga_default_vowels = sort keys %{$ht{SCRIPT_ABUDIGA_DEFAULT_VOWEL}->{$current_script}})
	&& ($abudiga_default_vowel = $abudiga_default_vowels[0])
	&& ($romanized_char =~ /^[bcdfghjklmnpqrstvwxyz]+$/i)) {
	my $new_roman = join("", $romanized_char, $abudiga_default_vowel);
	$this->add_node($new_roman, $i, $i+1, *chart_ht, "", "standard-unicode-based-romanization-plus-abudiga-default-vowel");
	# print STDERR " Abudiga add default vowel l.$line_number/$i: $new_roman\n" if $line_number =~ /^[12]$/;

	# Standard romanization
	} else {
	$node_id = $this->add_node($romanized_char, $i, $i+1, *chart_ht, "", "standard-unicode-based-romanization");
	}
	} else {
	$this->add_node($char, $i, $i+1, *chart_ht, "", "unexpected-original");
	}
	} elsif (defined($romanized_char = $this->romanize_char_at_position($i, $lang_code, $output_style, ht, chart_ht))
	&& ((length($romanized_char) <= 2)
	\|\| ($ht{UTF_TO_CHAR_ROMANIZATION}->{$char}))) { # or from unicode_overwrite_romanization table
	$romanized_char =~ s/^""$//;
	$this->add_node($romanized_char, $i, $i+1, *chart_ht, "", "romanized-without-character-name");
	} else {
	$this->add_node($char, $i, $i+1, *chart_ht, "", "unexpected-original-without-character-name");
	}
	}
	$i = $next_index;
	}

	$this->schwa_deletion(0, $n_characters, *chart_ht, $lang_code);
	$this->default_vowelize_tibetan(0, $n_characters, *chart_ht, $lang_code, $line_number) if $chart_ht{CHART_CONTAINS_SCRIPT}->{"Tibetan"};
	$this->assemble_numbers_in_chart(*chart_ht, $line_number);

	if ($return_chart_p) {
	} elsif ($return_offset_mappings_p) {
	($result, $offset_mappings, $new_char_offset, $new_rom_char_offset) = $this->best_romanized_string(0, $n_characters, *chart_ht, $control, $initial_char_offset, $initial_rom_char_offset);
	} else {
	$result = $this->best_romanized_string(0, $n_characters, *chart_ht) unless $return_chart_p;
	}

	if ($verbosePM) {
	my $logfile = "/nfs/isd/ulf/cgi-mt/amr-tmp/uroman-log.txt";
	$util->append_to_file($logfile, $log) if $log && (-r $logfile);
	}

	return ($result, $offset_mappings) if $return_offset_mappings_p;
	return *chart_ht if $return_chart_p;
	return $result;
	}

	sub string_to_json_string {
	local($this, $s) = @_;

	utf8::decode($s);
	my $j = JSON->new->utf8->encode([$s]);
	$j =~ s/^\[(.*)\]$/$1/;
	return $j;
	}

	sub chart_to_json_romanization_elements {
	local($this, $chart_start, $chart_end, *chart_ht, $line_number) = @_;

	my $result = "";
	my $start = $chart_start;
	my $end;
	while ($start < $chart_end) {
	$end = $this->find_end_of_rom_segment($start, $chart_end, *chart_ht);
	my @best_romanizations;
	if (($end && ($start < $end))
	&& (@best_romanizations = $this->best_romanizations($start, $end, *chart_ht))) {
	$orig_segment = $this->orig_string_at_span($start, $end, *chart_ht);
	$next_start = $end;
	} else {
	$orig_segment = $chart_ht{ORIG_CHAR}->{$start};
	@best_romanizations = ($orig);
	$next_start = $start + 1;
	}
	$exclusive_end = $end - 1;
	# $guarded_orig = $util->string_guard($orig_segment);
	$guarded_orig = $this->string_to_json_string($orig_segment);
	$result .= " { \"line\": $line_number, \"start\": $start, \"end\": $exclusive_end, \"orig\": $guarded_orig, \"roms\": [";
	foreach $i ((0 .. $#best_romanizations)) {
	my $rom = $best_romanizations[$i];
	# my $guarded_rom = $util->string_guard($rom);
	my $guarded_rom = $this->string_to_json_string($rom);
	$result .= " { \"rom\": $guarded_rom";
	# $result .= ", \"alt\": true" if $i >= 1;
	$result .= " }";
	$result .= "," if $i < $#best_romanizations;
	}
	$result .= " ] },\n";
	$start = $next_start;
	}
	return $result;
	}

	sub default_vowelize_tibetan {
	local($this, $chart_start, $chart_end, *chart_ht, $lang_code, $line_number) = @_;

	# my $verbose = ($line_number == 103);
	# print STDERR "\nStart default_vowelize_tibetan l.$line_number $chart_start-$chart_end\n" if $verbose;
	my $token_start = $chart_start;
	my $next_token_start = $chart_start;
	while (($token_start = $next_token_start) < $chart_end) {
	$next_token_start = $token_start + 1;

	next unless $chart_ht{CHAR_LETTER_PLUS}->{$token_start};
	my $current_script = $chart_ht{CHAR_SCRIPT}->{$token_start};
	next unless ($current_script eq "Tibetan");
	my $token_end = $chart_ht{LETTER_TOKEN_SEGMENT_START_TO_END}->{$token_start};
	next unless $token_end;
	next unless $token_end > $token_start;
	$next_token_start = $token_end;

	my $start = $token_start;
	my $end;
	my @node_ids = ();
	while ($start < $token_end) {
	$end = $this->find_end_of_rom_segment($start, $chart_end, *chart_ht);
	last unless $end && ($end > $start);
	my @alt_node_ids = sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}};
	last unless @alt_node_ids;
	push(@node_ids, $alt_node_ids[0]);
	$start = $end;
	}
	my $contains_vowel_p = 0;
	my @romanizations = ();
	foreach $node_id (@node_ids) {
	my $roman = $chart_ht{NODE_ROMAN}->{$node_id};
	$roman = "" unless defined($roman);
	push(@romanizations, $roman);
	$contains_vowel_p = 1 if $roman =~ /[aeiou]/i;
	}
	# print STDERR " old: $token_start-$token_end @romanizations\n" if $verbose;
	unless ($contains_vowel_p) {
	my $default_vowel_target_index;
	if ($#node_ids <= 1) {
	$default_vowel_target_index = 0;
	} elsif ($romanizations[$#romanizations] eq "s") {
	if ($romanizations[($#romanizations-1)] eq "y") {
	$default_vowel_target_index = $#romanizations-1;
	} else {
	$default_vowel_target_index = $#romanizations-2;
	}
	} else {
	$default_vowel_target_index = $#romanizations-1;
	}
	$romanizations[$default_vowel_target_index] .= "a";
	my $old_node_id = $node_ids[$default_vowel_target_index];
	my $old_start = $chart_ht{NODE_START}->{$old_node_id};
	my $old_end = $chart_ht{NODE_END}->{$old_node_id};
	my $old_roman = $chart_ht{NODE_ROMAN}->{$old_node_id};
	my $new_roman = $old_roman . "a";
	my $new_node_id = $this->add_node($new_roman, $old_start, $old_end, *chart_ht, "", "tibetan-default-vowel");
	$this->copy_slot_values($old_node_id, $new_node_id, *chart_id, "all");
	$chart_ht{NODE_TYPE}->{$old_node_id} = "backup"; # keep, but demote
	}
	if (($romanizations[0] eq "'")
	&& ($#romanizations >= 1)
	&& ($romanizations[1] =~ /^[o]$/)) {
	my $old_node_id = $node_ids[0];
	my $old_start = $chart_ht{NODE_START}->{$old_node_id};
	my $old_end = $chart_ht{NODE_END}->{$old_node_id};
	my $new_node_id = $this->add_node("", $old_start, $old_end, *chart_ht, "", "tibetan-delete-apostrophe");
	$this->copy_slot_values($old_node_id, $new_node_id, *chart_id, "all");
	$chart_ht{NODE_TYPE}->{$old_node_id} = "alt"; # keep, but demote
	}
	if (($#node_ids >= 1)
	&& ($romanizations[$#romanizations] =~ /^[bcdfghjklmnpqrstvwxz]+y$/)) {
	my $old_node_id = $node_ids[$#romanizations];
	my $old_start = $chart_ht{NODE_START}->{$old_node_id};
	my $old_end = $chart_ht{NODE_END}->{$old_node_id};
	my $old_roman = $chart_ht{NODE_ROMAN}->{$old_node_id};
	my $new_roman = $old_roman . "a";
	my $new_node_id = $this->add_node($new_roman, $old_start, $old_end, *chart_ht, "", "tibetan-syllable-final-vowel");
	$this->copy_slot_values($old_node_id, $new_node_id, *chart_id, "all");
	$chart_ht{NODE_TYPE}->{$old_node_id} = "alt"; # keep, but demote
	}
	foreach $old_node_id (@node_ids) {
	my $old_roman = $chart_ht{NODE_ROMAN}->{$old_node_id};
	next unless $old_roman =~ /-a/;
	my $old_start = $chart_ht{NODE_START}->{$old_node_id};
	my $old_end = $chart_ht{NODE_END}->{$old_node_id};
	my $new_roman = $old_roman;
	$new_roman =~ s/-a/a/;
	my $new_node_id = $this->add_node($new_roman, $old_start, $old_end, *chart_ht, "", "tibetan-syllable-delete-dash");
	$this->copy_slot_values($old_node_id, $new_node_id, *chart_id, "all");
	$chart_ht{NODE_TYPE}->{$old_node_id} = "alt"; # keep, but demote
	}
	}
	}

	sub schwa_deletion {
	local($this, $chart_start, $chart_end, *chart_ht, $lang_code) = @_;
	# delete word-final simple "a" in Devanagari (e.g. nepaala -> nepaal)
	# see Wikipedia article "Schwa deletion in Indo-Aryan languages"

	if ($chart_ht{CHART_CONTAINS_SCRIPT}->{"Devanagari"}) {
	my $script_start = $chart_start;
	my $next_script_start = $chart_start;
	while (($script_start = $next_script_start) < $chart_end) {
	$next_script_start = $script_start + 1;

	my $current_script = $chart_ht{CHAR_SCRIPT}->{$script_start};
	next unless ($current_script eq "Devanagari");
	my $script_end = $chart_ht{SCRIPT_SEGMENT_START_TO_END}->{$script_start};
	next unless $script_end;
	next unless $script_end - $script_start >= 2;
	$next_script_start = $script_end;
	my $end_node_id = $this->get_node_for_span($script_end-1, $script_end, *chart_ht);
	next unless $end_node_id;
	my $end_roman = $chart_ht{NODE_ROMAN}->{$end_node_id};
	next unless ($end_consonant) = ($end_roman =~ /^([bcdfghjklmnpqrstvwxz]+)a$/i);
	my $prev_node_id = $this->get_node_for_span($script_end-4, $script_end-1, *chart_ht)
	\|\| $this->get_node_for_span($script_end-3, $script_end-1, *chart_ht)
	\|\| $this->get_node_for_span($script_end-2, $script_end-1, *chart_ht);
	next unless $prev_node_id;
	my $prev_roman = $chart_ht{NODE_ROMAN}->{$prev_node_id};
	next unless $prev_roman =~ /[aeiou]/i;
	# TO DO: check further back for vowel (e.g. if $prev_roman eq "r" due to vowel cancelation)

	$chart_ht{NODE_TYPE}->{$end_node_id} = "alt"; # keep, but demote
	# print STDERR "* Schwa deletion " . ($script_end-1) . "-$script_end $end_roman->$end_consonant\n";
	$this->add_node($end_consonant, $script_end-1, $script_end, *chart_ht, "", "devanagari-with-deleted-final-schwa");
	}
	}
	}

	sub best_romanized_string {
	local($this, $chart_start, $chart_end, *chart_ht, $control, $orig_char_offset, $rom_char_offset) = @_;

	$control = "" unless defined($control);
	my $current_orig_char_offset = $orig_char_offset \|\| 0;
	my $current_rom_char_offset = $rom_char_offset \|\| 0;
	my $return_offset_mappings_p = ($control =~ /\breturn offset mappings\b/);
	my $result = "";
	my $start = $chart_start;
	my $end;
	my @char_offsets = ("$current_orig_char_offset:$current_rom_char_offset");
	while ($start < $chart_end) {
	$end = $this->find_end_of_rom_segment($start, $chart_end, *chart_ht);
	my $n_orig_chars_in_segment = 0;
	my $n_rom_chars_in_segment = 0;
	if ($end && ($start < $end)) {
	my @best_romanizations = $this->best_romanizations($start, $end, *chart_ht);
	my $best_romanization = (@best_romanizations) ? $best_romanizations[0] : undef;
	if (defined($best_romanization)) {
	$result .= $best_romanization;
	if ($return_offset_mappings_p) {
	$n_orig_chars_in_segment = $end-$start;
	$n_rom_chars_in_segment = $utf8->length_in_utf8_chars($best_romanization);
	}
	$start = $end;
	} else {
	my $best_romanization = $chart_ht{ORIG_CHAR}->{$start};
	$result .= $best_romanization;
	$start++;
	if ($return_offset_mappings_p) {
	$n_orig_chars_in_segment = 1;
	$n_rom_chars_in_segment = $utf8->length_in_utf8_chars($best_romanization);
	}
	}
	} else {
	my $best_romanization = $chart_ht{ORIG_CHAR}->{$start};
	$result .= $best_romanization;
	$start++;
	if ($return_offset_mappings_p) {
	$n_orig_chars_in_segment = 1;
	$n_rom_chars_in_segment = $utf8->length_in_utf8_chars($best_romanization);
	}
	}
	if ($return_offset_mappings_p) {
	my $new_orig_char_offset = $current_orig_char_offset + $n_orig_chars_in_segment;
	my $new_rom_char_offset = $current_rom_char_offset + $n_rom_chars_in_segment;
	my $offset_mapping = "$new_orig_char_offset:$new_rom_char_offset";
	push(@char_offsets, $offset_mapping);
	$current_orig_char_offset = $new_orig_char_offset;
	$current_rom_char_offset = $new_rom_char_offset;
	}
	}
	return ($result, join(",", @char_offsets), $current_orig_char_offset, $current_rom_char_offset) if $return_offset_mappings_p;
	return $result;
	}

	sub orig_string_at_span {
	local($this, $start, $end, *chart_ht) = @_;

	my $result = "";
	foreach $i (($start .. ($end-1))) {
	$result .= $chart_ht{ORIG_CHAR}->{$i};
	}
	return $result;
	}

	sub find_end_of_rom_segment {
	local($this, $start, $chart_end, *chart_ht) = @_;

	my @ends = sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}};
	my $end_index = $#ends;
	while (($end_index >= 0) && ($ends[$end_index] > $chart_end)) {
	$end_index--;
	}
	if (($end_index >= 0)
	&& defined($end = $ends[$end_index])
	&& ($start < $end)) {
	return $end;
	} else {
	return "";
	}
	}

	sub best_romanizations {
	local($this, $start, $end, *chart_ht) = @_;

	@regular_romanizations = ();
	@alt_romanizations = ();
	@backup_romanizations = ();

	foreach $node_id (sort { $a <=> $b } keys %{$chart_ht{NODES_STARTING_AND_ENDING_AT}->{$start}->{$end}}) {
	my $type = $chart_ht{NODE_TYPE}->{$node_id};
	my $roman = $chart_ht{NODE_ROMAN}->{$node_id};
	if (! defined($roman)) {
	# ignore
	} elsif (($type eq "backup") && ! defined($backup_romanization)) {
	push(@backup_romanizations, $roman) unless $util->member($roman, @backup_romanizations);
	} elsif (($type eq "alt") && ! defined($alt_romanization)) {
	push(@alt_romanizations, $roman) unless $util->member($roman, @alt_romanizations);
	} else {
	push(@regular_romanizations, $roman) unless $util->member($roman, @regular_romanizations);
	}
	}
	@regular_alt_romanizations = sort @regular_romanizations;
	foreach $alt_romanization (sort @alt_romanizations) {
	push(@regular_alt_romanizations, $alt_romanization) unless $util->member($alt_romanization, @regular_alt_romanizations);
	}
	return @regular_alt_romanizations if @regular_alt_romanizations;
	return sort @backup_romanizations;
	}

	sub join_alt_romanizations_for_viz {
	local($this, @list) = @_;

	my @viz_romanizations = ();

	foreach $alt_rom (@list) {
	if ($alt_rom eq "") {
	push(@viz_romanizations, "-");
	} else {
	push(@viz_romanizations, $alt_rom);
	}
	}
	return join(", ", @viz_romanizations);
	}

	sub markup_orig_rom_strings {
	local($this, $chart_start, $chart_end, ht, chart_ht, *pinyin_ht, $last_group_id_index) = @_;

	my $marked_up_rom = "";
	my $marked_up_orig = "";
	my $start = $chart_start;
	my $end;
	while ($start < $chart_end) {
	my $segment_start = $start;
	my $segment_end = $start+1;
	my $end = $this->find_end_of_rom_segment($start, $chart_end, *chart_ht);
	my $rom_segment = "";
	my $orig_segment = "";
	my $rom_title = "";
	my $orig_title = "";
	my $contains_alt_romanizations = 0;
	if ($end) {
	$segment_end = $end;
	my @best_romanizations = $this->best_romanizations($start, $end, *chart_ht);
	my $best_romanization = (@best_romanizations) ? $best_romanizations[0] : undef;
	if (defined($best_romanization)) {
	$rom_segment .= $best_romanization;
	$orig_segment .= $this->orig_string_at_span($start, $end, *chart_ht);
	$segment_end = $end;
	if ($#best_romanizations >= 1) {
	$rom_title .= $util->guard_html("Alternative romanizations: " . $this->join_alt_romanizations_for_viz(@best_romanizations) . "\n");
	$contains_alt_romanizations = 1;
	}
	} else {
	my $segment = $this->orig_string_at_span($start, $start+1, *chart_ht);
	$rom_segment .= $segment;
	$orig_segment .= $segment;
	$segment_end = $start+1;
	}
	$start = $segment_end;
	} else {
	$rom_segment .= $chart_ht{ORIG_CHAR}->{$start};
	$orig_segment .= $this->orig_string_at_span($start, $start+1, *chart_ht);
	$segment_end = $start+1;
	$start = $segment_end;
	}
	my $next_char = $chart_ht{ORIG_CHAR}->{$segment_end};
	my $next_char_is_combining_p = $this->char_is_combining_char($next_char, *ht);
	while ($next_char_is_combining_p
	&& ($segment_end < $chart_end)
	&& ($end = $this->find_end_of_rom_segment($segment_end, $chart_end, *chart_ht))
	&& ($end > $segment_end)
	&& (@best_romanizations = $this->best_romanizations($segment_end, $end, *chart_ht))
	&& defined($best_romanization = $best_romanizations[0])) {
	$orig_segment .= $this->orig_string_at_span($segment_end, $end, *chart_ht);
	$rom_segment .= $best_romanization;
	if ($#best_romanizations >= 1) {
	$rom_title .= $util->guard_html("Alternative romanizations: " . $this->join_alt_romanizations_for_viz(@best_romanizations) . "\n");
	$contains_alt_romanizations = 1;
	}
	$segment_end = $end;
	$start = $segment_end;
	$next_char = $chart_ht{ORIG_CHAR}->{$segment_end};
	$next_char_is_combining_p = $this->char_is_combining_char($next_char, *ht);
	}
	foreach $i (($segment_start .. ($segment_end-1))) {
	$orig_title .= "+‎ ‎" unless $orig_title eq "";
	my $char = $chart_ht{ORIG_CHAR}->{$i};
	my $numeric = $ht{UTF_TO_NUMERIC}->{$char};
	$numeric = "" unless defined($numeric);
	my $pic_descr = $ht{UTF_TO_PICTURE_DESCR}->{$char};
	$pic_descr = "" unless defined($pic_descr);
	if ($char =~ /^\xE4\xB7[\x80-\xBF]$/) {
	$orig_title .= "$char_name\n";
	} elsif (($char =~ /^[\xE3-\xE9][\x80-\xBF]{2,2}$/) && $chinesePM->string_contains_utf8_cjk_unified_ideograph_p($char)) {
	my $unicode = $utf8->utf8_to_unicode($char);
	$orig_title .= "CJK Unified Ideograph U+" . (uc sprintf("%04x", $unicode)) . "\n";
	$orig_title .= "Chinese: $tonal_translit\n" if $tonal_translit = $chinesePM->tonal_pinyin($char, *pinyin_ht, "");
	$orig_title .= "Number: $numeric\n" if $numeric =~ /\d/;
	} elsif ($char_name = $ht{UTF_TO_CHAR_NAME}->{$char}) {
	$orig_title .= "$char_name\n";
	$orig_title .= "Number: $numeric\n" if $numeric =~ /\d/;
	$orig_title .= "Picture: $pic_descr\n" if $pic_descr =~ /\S/;
	} else {
	my $unicode = $utf8->utf8_to_unicode($char);
	if (($unicode >= 0xAC00) && ($unicode <= 0xD7A3)) {
	$orig_title .= "Hangul syllable U+" . (uc sprintf("%04x", $unicode)) . "\n";
	} else {
	$orig_title .= "Unicode character U+" . (uc sprintf("%04x", $unicode)) . "\n";
	}
	}
	}
	(@non_ascii_roms) = ($rom_segment =~ /([\xC0-\xFF][\x80-\xBF]*)/g);
	foreach $char (@non_ascii_roms) {
	my $char_name = $ht{UTF_TO_CHAR_NAME}->{$char};
	my $unicode = $utf8->utf8_to_unicode($char);
	my $unicode_s = "U+" . (uc sprintf("%04x", $unicode));
	if ($char_name) {
	$rom_title .= "$char_name\n";
	} else {
	$rom_title .= "$unicode_s\n";
	}
	}
	$last_group_id_index++;
	$rom_title =~ s/\s*$//;
	$rom_title =~ s/\n/ /g;
	$orig_title =~ s/\s*$//;
	$orig_title =~ s/\n/ ‎/g;
	$orig_title = "‭" . $orig_title . "‬";
	my $rom_title_clause = ($rom_title eq "") ? "" : " title=\"$rom_title\"";
	my $orig_title_clause = ($orig_title eq "") ? "" : " title=\"$orig_title\"";
	my $alt_rom_clause = ($contains_alt_romanizations) ? "border-bottom:1px dotted;" : "";
	$marked_up_rom .= "<span id=\"span-$last_group_id_index-1\" onmouseover=\"highlight_elems('span-$last_group_id_index','1');\" onmouseout=\"highlight_elems('span-$last_group_id_index','0');\" style=\"color:#00BB00;$alt_rom_clause\"$rom_title_clause>" . $util->guard_html($rom_segment) . "<\/span>";
	$marked_up_orig .= "<span id=\"span-$last_group_id_index-2\" onmouseover=\"highlight_elems('span-$last_group_id_index','1');\" onmouseout=\"highlight_elems('span-$last_group_id_index','0');\"$orig_title_clause>" . $util->guard_html($orig_segment) . "<\/span>";
	if (($last_char = $chart_ht{ORIG_CHAR}->{($segment_end-1)})
	&& ($last_char_name = $ht{UTF_TO_CHAR_NAME}->{$last_char})
	&& ($last_char_name =~ /^(FULLWIDTH COLON\|FULLWIDTH COMMA\|FULLWIDTH RIGHT PARENTHESIS\|IDEOGRAPHIC COMMA\|IDEOGRAPHIC FULL STOP\|RIGHT CORNER BRACKET\|BRAILLE PATTERN BLANK\|TIBETAN MARK .*)$/)) {
	$marked_up_orig .= "<wbr>";
	$marked_up_rom .= "<wbr>";
	}
	}
	return ($marked_up_rom, $marked_up_orig, $last_group_id_index);
	}

	sub romanizations_with_alternatives {
	local($this, ht, chart_ht, *pinyin_ht, $chart_start, $chart_end) = @_;

	$chart_start = 0 unless defined($chart_start);
	$chart_end = $chart_ht{N_CHARS} unless defined($chart_end);
	my $result = "";
	my $start = $chart_start;
	my $end;
	# print STDOUT "romanizations_with_alternatives $chart_start-$chart_end\n";
	while ($start < $chart_end) {
	my $segment_start = $start;
	my $segment_end = $start+1;
	my $end = $this->find_end_of_rom_segment($start, $chart_end, *chart_ht);
	my $rom_segment = "";
	# print STDOUT " $start-$end\n";
	if ($end) {
	$segment_end = $end;
	my @best_romanizations = $this->best_romanizations($start, $end, *chart_ht);
	# print STDOUT " $start-$end @best_romanizations\n";
	if (@best_romanizations) {
	if ($#best_romanizations == 0) {
	$rom_segment .= $best_romanizations[0];
	} else {
	$rom_segment .= "{" . join("\|", @best_romanizations) . "}";
	}
	$segment_end = $end;
	} else {
	my $segment = $this->orig_string_at_span($start, $start+1, *chart_ht);
	$rom_segment .= $segment;
	$segment_end = $start+1;
	}
	$start = $segment_end;
	} else {
	$rom_segment .= $chart_ht{ORIG_CHAR}->{$start};
	$segment_end = $start+1;
	$start = $segment_end;
	}
	# print STDOUT " $start-$end ** $rom_segment\n";
	$result .= $rom_segment;
	}
	return $result;
	}

	sub quick_romanize {
	local($this, $s, $lang_code, *ht) = @_;

	my $result = "";
	my @chars = $utf8->split_into_utf8_characters($s, "return only chars", *empty_ht);
	while (@chars) {
	my $found_match_in_table_p = 0;
	foreach $string_length (reverse(1..4)) {
	next if ($string_length-1) > $#chars;
	$multi_char_substring = join("", @chars[0..($string_length-1)]);
	my @mappings = keys %{$ht{UTF_CHAR_MAPPING_LANG_SPEC}->{$lang_code}->{$multi_char_substring}};
	@mappings = keys %{$ht{UTF_CHAR_MAPPING}->{$multi_char_substring}} unless @mappings;
	if (@mappings) {
	my $mapping = $mappings[0];
	$result .= $mapping;
	foreach $_ ((1 .. $string_length)) {
	shift @chars;
	}
	$found_match_in_table_p = 1;
	last;
	}
	}
	unless ($found_match_in_table_p) {
	$result .= $chars[0];
	shift @chars;
	}
	}
	return $result;
	}

	sub char_is_combining_char {
	local($this, $c, *ht) = @_;

	return 0 unless $c;
	my $category = $ht{UTF_TO_CAT}->{$c};
	return 0 unless $category;
	return $category =~ /^M/;
	}

	sub mark_up_string_for_mouse_over {
	local($this, $s, ht, $control, pinyin_ht) = @_;

	$control = "" unless defined($control);
	$no_ascii_p = ($control =~ /NO-ASCII/);
	my $result = "";
	@chars = $utf8->split_into_utf8_characters($s, "return only chars", *empty_ht);
	while (@chars) {
	$char = shift @chars;
	$numeric = $ht{UTF_TO_NUMERIC}->{$char};
	$numeric = "" unless defined($numeric);
	$pic_descr = $ht{UTF_TO_PICTURE_DESCR}->{$char};
	$pic_descr = "" unless defined($pic_descr);
	$next_char = ($#chars >= 0) ? $chars[0] : "";
	$next_char_is_combining_p = $this->char_is_combining_char($next_char, *ht);
	if ($no_ascii_p
	&& ($char =~ /^[\x00-\x7F]*$/)
	&& ! $next_char_is_combining_p) {
	$result .= $util->guard_html($char);
	} elsif (($char =~ /^[\xE3-\xE9][\x80-\xBF]{2,2}$/) && $chinesePM->string_contains_utf8_cjk_unified_ideograph_p($char)) {
	$unicode = $utf8->utf8_to_unicode($char);
	$title = "CJK Unified Ideograph U+" . (uc sprintf("%04x", $unicode));
	$title .= " Chinese: $tonal_translit" if $tonal_translit = $chinesePM->tonal_pinyin($char, *pinyin_ht, "");
	$title .= " Number: $numeric" if $numeric =~ /\d/;
	$result .= "<span title=\"$title\">" . $util->guard_html($char) . "<\/span>";
	} elsif ($char_name = $ht{UTF_TO_CHAR_NAME}->{$char}) {
	$title = $char_name;
	$title .= " Number: $numeric" if $numeric =~ /\d/;
	$title .= " Picture: $pic_descr" if $pic_descr =~ /\S/;
	$char_plus = $char;
	while ($next_char_is_combining_p) {
	# combining marks (Mc:non-spacing, Mc:spacing combining, Me: enclosing)
	$next_char_name = $ht{UTF_TO_CHAR_NAME}->{$next_char};
	$title .= " + $next_char_name";
	$char = shift @chars;
	$char_plus .= $char;
	$next_char = ($#chars >= 0) ? $chars[0] : "";
	$next_char_is_combining_p = $this->char_is_combining_char($next_char, *ht);
	}
	$result .= "<span title=\"$title\">" . $util->guard_html($char_plus) . "<\/span>";
	$result .= "<wbr>" if $char_name =~ /^(FULLWIDTH COLON\|FULLWIDTH COMMA\|FULLWIDTH RIGHT PARENTHESIS\|IDEOGRAPHIC COMMA\|IDEOGRAPHIC FULL STOP\|RIGHT CORNER BRACKET)$/;
	} elsif (($unicode = $utf8->utf8_to_unicode($char))
	&& ($unicode >= 0xAC00) && ($unicode <= 0xD7A3)) {
	$title = "Hangul syllable U+" . (uc sprintf("%04x", $unicode));
	$result .= "<span title=\"$title\">" . $util->guard_html($char) . "<\/span>";
	} else {
	$result .= $util->guard_html($char);
	}
	}
	return $result;
	}

	sub romanize_char_at_position_incl_multi {
	local($this, $i, $lang_code, $output_style, ht, chart_ht) = @_;

	my $char = $chart_ht{ORIG_CHAR}->{$i};
	return "" unless defined($char);
	my @mappings = keys %{$ht{UTF_CHAR_MAPPING_LANG_SPEC}->{$lang_code}->{$char}};
	return $mappings[0] if @mappings;
	@mappings = keys %{$ht{UTF_CHAR_MAPPING}->{$char}};
	return $mappings[0] if @mappings;
	return $this->romanize_char_at_position($i, $lang_code, $output_style, ht, chart_ht);
	}

	sub romanize_char_at_position {
	local($this, $i, $lang_code, $output_style, ht, chart_ht) = @_;

	my $char = $chart_ht{ORIG_CHAR}->{$i};
	return "" unless defined($char);
	return $char if $char =~ /^[\x00-\x7F]$/; # ASCII
	my $romanization = $ht{UTF_TO_CHAR_ROMANIZATION}->{$char};
	return $romanization if $romanization;
	my $char_name = $chart_ht{CHAR_NAME}->{$i};
	$romanization = $this->romanize_charname($char_name, $lang_code, $output_style, *ht, $char);
	$ht{SUSPICIOUS_ROMANIZATION}->{$char_name}->{$romanization}
	= ($ht{SUSPICIOUS_ROMANIZATION}->{$char_name}->{$romanization} \|\| 0) + 1
	unless (length($romanization) < 4)
	\|\| ($romanization =~ /\s/)
	\|\| ($romanization =~ /^[bcdfghjklmnpqrstvwxyz]{2,3}[aeiou]-$/) # Khmer ngo-/nyo-/pho- OK
	\|\| ($romanization =~ /^[bcdfghjklmnpqrstvwxyz]{2,2}[aeiougw][aeiou]{1,2}$/) # Canadian, Ethiopic syllable OK
	\|\| ($romanization =~ /^(allah\|bbux\|nyaa\|nnya\|quuv\|rrep\|shch\|shur\|syrx)$/i) # Arabic; Yi; Ethiopic syllable nyaa; Cyrillic letter shcha
	\|\| (($char_name =~ /^(YI SYLLABLE\|VAI SYLLABLE\|ETHIOPIC SYLLABLE\|CANADIAN SYLLABICS\|CANADIAN SYLLABICS CARRIER)\s+(\S+)$/) && (length($romanization) <= 5));
	# print STDERR "romanize_char_at_position $i $char_name :: $romanization\n" if $char_name =~ /middle/i;
	return $romanization;
	}

	sub romanize_charname {
	local($this, $char_name, $lang_code, $output_style, *ht, $char) = @_;

	my $cached_result = $ht{ROMANIZE_CHARNAME}->{$char_name}->{$lang_code}->{$output_style};
	# print STDERR "(C) romanize_charname($char_name): $cached_result\n" if $cached_result && ($char_name =~ /middle/i);
	return $cached_result if defined($cashed_result);
	$orig_char_name = $char_name;
	$char_name =~ s/^.* LETTER\s+([A-Z]+)-\d+$/$1/; # HENTAIGANA LETTER A-3
	$char_name =~ s/^.* LETTER\s+//;
	$char_name =~ s/^.* SYLLABLE\s+B\d\d\d\s+//; # Linear B syllables
	$char_name =~ s/^.* SYLLABLE\s+//;
	$char_name =~ s/^.* SYLLABICS\s+//;
	$char_name =~ s/^.* LIGATURE\s+//;
	$char_name =~ s/^.* VOWEL SIGN\s+//;
	$char_name =~ s/^.* CONSONANT SIGN\s+//;
	$char_name =~ s/^.* CONSONANT\s+//;
	$char_name =~ s/^.* VOWEL\s+//;
	$char_name =~ s/ WITH .*$//;
	$char_name =~ s/ WITHOUT .*$//;
	$char_name =~ s/\s+(ABOVE\|AGUNG\|BAR\|BARREE\|BELOW\|CEDILLA\|CEREK\|DIGRAPH\|DOACHASHMEE\|FINAL FORM\|GHUNNA\|GOAL\|INITIAL FORM\|ISOLATED FORM\|KAWI\|LELET\|LELET RASWADI\|LONSUM\|MAHAPRANA\|MEDIAL FORM\|MURDA\|MURDA MAHAPRANA\|REVERSED\|ROTUNDA\|SASAK\|SUNG\|TAM\|TEDUNG\|TYPE ONE\|TYPE TWO\|WOLOSO)\s*$//;
	$char_name =~ s/^([A-Z]+)\d+$/$1/; # Linear B syllables etc.
	foreach $_ ((1 .. 3)) {
	$char_name =~ s/^.*\b(?:ABKHASIAN\|ACADEMY\|AFRICAN\|AIVILIK\|AITON\|AKHMIMIC\|ALEUT\|ALI GALI\|ALPAPRAANA\|ALTERNATE\|ALTERNATIVE\|AMBA\|ARABIC\|ARCHAIC\|ASPIRATED\|ATHAPASCAN\|BASELINE\|BLACKLETTER\|BARRED\|BASHKIR\|BERBER\|BHATTIPROLU\|BIBLE-CREE\|BIG\|BINOCULAR\|BLACKFOOT\|BLENDED\|BOTTOM\|BROAD\|BROKEN\|CANDRA\|CAPITAL\|CARRIER\|CHILLU\|CLOSE\|CLOSED\|COPTIC\|CROSSED\|CRYPTOGRAMMIC\|CURLED\|CURLY\|CYRILLIC\|DANTAJA\|DENTAL\|DIALECT-P\|DIAERESIZED\|DOTLESS\|DOUBLE\|DOUBLE-STRUCK\|EASTERN PWO KAREN\|EGYPTOLOGICAL\|FARSI\|FINAL\|FLATTENED\|GLOTTAL\|GREAT\|GREEK\|HALF\|HIGH\|INITIAL\|INSULAR\|INVERTED\|IOTIFIED\|JONA\|KANTAJA\|KASHMIRI\|KHAKASSIAN\|KHAMTI\|KHANDA\|KINNA\|KIRGHIZ\|KOMI\|L-SHAPED\|LATINATE\|LITTLE\|LONG\|LONG-LEGGED\|LOOPED\|LOW\|MAHAAPRAANA\|MALAYALAM\|MANCHU\|MANDAILING\|MATHEMATICAL\|MEDIAL\|MIDDLE-WELSH\|MON\|MONOCULAR\|MOOSE-CREE\|MULTIOCULAR\|MUURDHAJA\|N-CREE\|NARROW\|NASKAPI\|NDOLE\|NEUTRAL\|NIKOLSBURG\|NORTHERN\|NUBIAN\|NUNAVIK\|NUNAVUT\|OJIBWAY\|OLD\|OPEN\|ORKHON\|OVERLONG\|PALI\|PERSIAN\|PHARYNGEAL\|PRISHTHAMATRA\|R-CREE\|REDUPLICATION\|REVERSED\|ROMANIAN\|ROUND\|ROUNDED\|RUDIMENTA\|RUMAI PALAUNG\|SANSKRIT\|SANYAKA\|SARA\|SAYISI\|SCRIPT\|SEBATBEIT\|SEMISOFT\|SGAW KAREN\|SHAN\|SHARP\|SHWE PALAUNG\|SHORT\|SIBE\|SIDEWAYS\|SIMALUNGUN\|SMALL\|SOGDIAN\|SOFT\|SOUTH-SLAVEY\|SOUTHERN\|SPIDERY\|STIRRUP\|STRAIGHT\|STRETCHED\|SUBSCRIPT\|SWASH\|TAI LAING\|TAILED\|TAILLESS\|TAALUJA\|TH-CREE\|TALL\|THREE-LEGGED\|TURNED\|TODO\|TOP\|TROKUTASTI\|TUAREG\|UKRAINIAN\|UNBLENDED\|VISIGOTHIC\|VOCALIC\|VOICED\|VOICELESS\|VOLAPUK\|WAVY\|WESTERN PWO KAREN\|WEST-CREE\|WESTERN\|WIDE\|WOODS-CREE\|Y-CREE\|YENISEI\|YIDDISH)\s+//;
	}
	$char_name =~ s/\s+(ABOVE\|AGUNG\|BAR\|BARREE\|BELOW\|CEDILLA\|CEREK\|DIGRAPH\|DOACHASHMEE\|FINAL FORM\|GHUNNA\|GOAL\|INITIAL FORM\|ISOLATED FORM\|KAWI\|LELET\|LELET RASWADI\|LONSUM\|MAHAPRANA\|MEDIAL FORM\|MURDA\|MURDA MAHAPRANA\|REVERSED\|ROTUNDA\|SASAK\|SUNG\|TAM\|TEDUNG\|TYPE ONE\|TYPE TWO\|WOLOSO)\s*$//;
	if ($char_name =~ /THAI CHARACTER/) {
	$char_name =~ s/^THAI CHARACTER\s+//;
	if ($char =~ /^\xE0\xB8[\x81-\xAE]/) {
	# Thai consonants
	$char_name =~ s/^([^AEIOU])./$1/i;
	} elsif ($char_name =~ /^SARA [AEIOU]/) {
	# Thai vowels
	$char_name =~ s/^SARA\s+//;
	} else {
	$char_name = $char;
	}
	}
	if ($orig_char_name =~ /(HIRAGANA LETTER\|KATAKANA LETTER\|SYLLABLE\|LIGATURE)/) {
	$char_name = lc $char_name;
	} elsif ($char_name =~ /\b(ANUSVARA\|ANUSVARAYA\|NIKAHIT\|SIGN BINDI\|TIPPI)\b/) {
	$char_name = "+m";
	} elsif ($char_name =~ /\bSCHWA\b/) {
	$char_name = "e";
	} elsif ($char_name =~ /\bIOTA\b/) {
	$char_name = "i";
	} elsif ($char_name =~ /\s/) {
	} elsif ($orig_char_name =~ /KHMER LETTER/) {
	$char_name .= "-";
	} elsif ($orig_char_name =~ /CHEROKEE LETTER/) {
	# use whole letter as is
	} elsif ($orig_char_name =~ /KHMER INDEPENDENT VOWEL/) {
	$char_name =~ s/q//;
	} elsif ($orig_char_name =~ /LETTER/) {
	$char_name =~ s/^[AEIOU]+([^AEIOU]+)$/$1/i;
	$char_name =~ s/^([^-AEIOUY]+)[AEIOU].*/$1/i;
	$char_name =~ s/^(Y)[AEIOU]./$1/i if $orig_char_name =~ /\b(?:BENGALI\|DEVANAGARI\|GURMUKHI\|GUJARATI\|KANNADA\|MALAYALAM\|MODI\|MYANMAR\|ORIYA\|TAMIL\|TELUGU\|TIBETAN)\b.\bLETTER YA\b/;
	$char_name =~ s/^(Y[AEIOU]+)[^AEIOU].*$/$1/i;
	$char_name =~ s/^([AEIOU]+)[^AEIOU]+[AEIOU].*/$1/i;
	}

	my $result = ($orig_char_name =~ /\bCAPITAL\b/) ? (uc $char_name) : (lc $char_name);
	# print STDERR "(R) romanize_charname($orig_char_name): $result\n" if $orig_char_name =~ /middle/i;
	$ht{ROMANIZE_CHARNAME}->{$char_name}->{$lang_code}->{$output_style} = $result;
	return $result;
	}

	sub assemble_numbers_in_chart {
	local($this, *chart_ht, $line_number) = @_;

	foreach $start (sort { $a <=> $b } keys %{$chart_ht{COMPLEX_NUMERIC_START_END}}) {
	my $end = $chart_ht{COMPLEX_NUMERIC_START_END}->{$start};
	my @numbers = ();
	foreach $i (($start .. ($end-1))) {
	my $orig_char = $chart_ht{ORIG_CHAR}->{$i};
	my $node_id = $this->get_node_for_span_with_slot($i, $i+1, "numeric-value", *chart_id);
	if (defined($node_id)) {
	my $number = $chart_ht{NODE_ROMAN}->{$node_id};
	if (defined($number)) {
	push(@numbers, $number);
	} elsif ($orig_char =~ /^[.,]$/) { # decimal point, comma separator
	push(@numbers, $orig_char);
	} else {
	print STDERR "Found no romanization for node_id $node_id ($i-" . ($i+1) . ") in assemble_numbers_in_chart\n" if $verbosePM;
	}
	} else {
	print STDERR "Found no node_id for span $i-" . ($i+1) . " in assemble_numbers_in_chart\n" if $verbosePM;
	}
	}
	my $complex_number = $this->assemble_number(join("\xC2\xB7", @numbers), $line_number);
	# print STDERR "assemble_numbers_in_chart l.$line_number $start-$end $complex_number (@numbers)\n";
	$this->add_node($complex_number, $start, $end, *chart_ht, "", "complex-number");
	}
	}

	sub assemble_number {
	local($this, $s, $line_number) = @_;
	# e.g. 10 9 100 7 10 8 = 1978

	my $middot = "\xC2\xB7";
	my @tokens = split(/$middot/, $s); # middle dot U+00B7
	my $i = 0;
	my @orig_tokens = @tokens;

	# assemble single digit numbers, e.g. 1 7 5 -> 175
	while ($i < $#tokens) {
	if ($tokens[$i] =~ /^\d$/) {
	my $j = $i+1;
	while (($j <= $#tokens) && ($tokens[$j] =~ /^[0-9.,]$/)) {
	$j++;
	}
	$j--;
	if ($j>$i) {
	my $new_token = join("", @tokens[$i .. $j]);
	$new_token =~ s/,//g;
	splice(@tokens, $i, $j-$i+1, $new_token);
	}
	}
	$i++;
	}

	foreach $power ((10, 100, 1000, 10000, 100000, 1000000, 100000000, 1000000000, 1000000000000)) {
	for (my $i=0; $i <= $#tokens; $i++) {
	if ($tokens[$i] == $power) {
	if (($i > 0) && ($tokens[($i-1)] < $power)) {
	splice(@tokens, $i-1, 2, ($tokens[($i-1)] * $tokens[$i]));
	$i--;
	if (($i < $#tokens) && ($tokens[($i+1)] < $power)) {
	splice(@tokens, $i, 2, ($tokens[$i] + $tokens[($i+1)]));
	$i--;
	}
	}
	}
	# 400 30 (e.g. Egyptian)
	my $gen_pattern = $power;
	$gen_pattern =~ s/^1/\[1-9\]/;
	if (($tokens[$i] =~ /^$gen_pattern$/) && ($i < $#tokens) && ($tokens[($i+1)] < $power)) {
	splice(@tokens, $i, 2, ($tokens[$i] + $tokens[($i+1)]));
	$i--;
	}
	}
	last if $#tokens == 0;
	}
	my $result = join($middot, @tokens);
	if ($verbosePM) {
	my $logfile = "/nfs/isd/ulf/cgi-mt/amr-tmp/uroman-number-log.txt";
	$util->append_to_file($logfile, "$s -> $result\n") if -r $logfile;
	# print STDERR " assemble number l.$line_number @orig_tokens -> $result\n" if $line_number == 43;
	}
	return $result;
	}

	1;