]>
Dogcows Code - chaz/tar/blob - src/utf8.c
1 /* Charset handling for GNU tar.
3 Copyright (C) 2004 Free Software Foundation, Inc.
5 This program is free software; you can redistribute it and/or modify it
6 under the terms of the GNU General Public License as published by the
7 Free Software Foundation; either version 2, or (at your option) any later
10 This program is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
13 Public License for more details.
15 You should have received a copy of the GNU General Public License along
16 with this program; if not, write to the Free Software Foundation, Inc.,
17 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */
28 char *lang
; /* Language code */
29 char *terr
; /* Territory code */
30 char *charset
; /* Corresponding charset */
33 /* The list of language codes defined in ISO 639 with the corresponding
34 default character sets.
38 1) The list must be ordered by:
39 a) lang field in ascending order
40 b) terr field in descending order.
41 NULL fields are considered less than non-null ones.
42 2) Many entries have NULL charset fields. Please help fill them!
43 3) The "default" character set for a given language is a matter
44 of preference. Possibly the table should contain a *list* of
45 possible character sets.
46 4) LC_ALL "modifier" field is not taken into account */
48 static struct langtab langtab
[] = {
49 { "C", NULL
, "ASCII"},
50 { "POSIX", NULL
, "ASCII" },
51 { "aa", NULL
, NULL
}, /* Afar */
52 { "ab", NULL
, NULL
}, /* Abkhazian */
53 { "ae", NULL
, NULL
}, /* Avestan */
54 { "af", NULL
, "iso-8859-1"}, /* Afrikaans */
55 { "am", NULL
, "UTF-8"}, /* Amharic */
56 { "ar", NULL
, "iso-8859-6"}, /* Arabic */
57 { "as", NULL
, NULL
}, /* Assamese */
58 { "ay", NULL
, "iso-8859-1"}, /* Aymara */
59 { "az", NULL
, NULL
}, /* Azerbaijani */
60 { "ba", NULL
, NULL
}, /* Bashkir */
61 { "be", NULL
, "UTF-8"}, /* Byelorussian; Belarusian */
62 { "bg", NULL
, "iso-8859-5"}, /* Bulgarian */
63 { "bh", NULL
, NULL
}, /* Bihari */
64 { "bi", NULL
, NULL
}, /* Bislama */
65 { "bn", NULL
, NULL
}, /* Bengali; Bangla */
66 { "bo", NULL
, NULL
}, /* Tibetan */
67 { "br", NULL
, "iso-8859-1"}, /* Breton: 1,5,8,9 */
68 { "bs", NULL
, NULL
}, /* Bosnian */
69 { "ca", NULL
, "iso-8859-1"}, /* Catalan: 1,5,8,9 */
70 { "ce", NULL
, NULL
}, /* Chechen */
71 { "ch", NULL
, NULL
}, /* Chamorro */
72 { "co", NULL
, "iso-8859-1"}, /* Corsican */
73 { "cs", NULL
, "iso-8859-2"}, /* Czech */
74 { "cu", NULL
, NULL
}, /* Church Slavic */
75 { "cv", NULL
, NULL
}, /* Chuvash */
76 { "cy", NULL
, "iso-8859-1"}, /* Welsh */
77 { "da", NULL
, "iso-8859-1"}, /* Danish: 4-9 */
78 { "de", NULL
, "iso-8859-1"}, /* German */
79 { "dz", NULL
, NULL
}, /* Dzongkha; Bhutani */
80 { "el", NULL
, "iso-8859-7"}, /* Greek */
81 { "en", NULL
, "iso-8859-1"}, /* English */
82 { "eo", NULL
, "iso-8859-3"}, /* Esperanto */
83 { "es", NULL
, "iso-8859-1"}, /* Spanish */
84 { "et", NULL
, "iso-8859-15"}, /* Estonian: 6,7,9 */
85 { "eu", NULL
, "iso-8859-1"}, /* Basque: 5,8,9 */
86 { "fa", NULL
, "UTF-8"}, /* Persian */
87 { "fi", NULL
, "iso-8859-15"}, /* Finnish */
88 { "fj", NULL
, NULL
}, /* Fijian; Fiji */
89 { "fo", NULL
, "iso-8859-1"}, /* Faroese: 6,9 */
90 { "fr", NULL
, "iso-8859-1"}, /* French */
91 { "fy", NULL
, "iso-8859-1"}, /* Frisian */
92 { "ga", NULL
, "iso-8859-14"}, /* Irish */
93 { "gd", NULL
, "iso-8859-14" }, /* Scots; Gaelic */
94 { "gl", NULL
, NULL
}, /* Gallegan; Galician */
95 { "gn", NULL
, NULL
}, /* Guarani */
96 { "gu", NULL
, NULL
}, /* Gujarati */
97 { "gv", NULL
, "iso-8859-14"}, /* Manx */
98 { "ha", NULL
, NULL
}, /* Hausa (?) */
99 { "he", NULL
, "iso-8859-8" }, /* Hebrew */
100 { "hi", NULL
, NULL
}, /* Hindi */
101 { "ho", NULL
, NULL
}, /* Hiri Motu */
102 { "hr", NULL
, "iso-8859-2"}, /* Croatian: 10 */
103 { "hu", NULL
, "iso-8859-2"}, /* Hungarian */
104 { "hy", NULL
, NULL
}, /* Armenian */
105 { "hz", NULL
, NULL
}, /* Herero */
106 { "id", NULL
, "iso-8859-1"}, /* Indonesian (formerly in) */
107 { "ia", NULL
, NULL
}, /* Interlingua */
108 { "ie", NULL
, NULL
}, /* Interlingue */
109 { "ik", NULL
, NULL
}, /* Inupiak */
110 { "io", NULL
, NULL
}, /* Ido */
111 { "is", NULL
, "iso-8859-1"}, /* Icelandic */
112 { "it", NULL
, "iso-8859-1"}, /* Italian */
113 { "iu", NULL
, NULL
}, /* Inuktitut */
114 { "ja", NULL
, "EUC-JP"}, /* Japanese */
115 { "jv", NULL
, NULL
}, /* Javanese */
116 { "ka", NULL
, NULL
}, /* Georgian */
117 { "ki", NULL
, NULL
}, /* Kikuyu */
118 { "kj", NULL
, NULL
}, /* Kuanyama */
119 { "kk", NULL
, NULL
}, /* Kazakh */
120 { "kl", NULL
, "iso-8859-1"}, /* Kalaallisut; Greenlandic */
121 { "km", NULL
, NULL
}, /* Khmer; Cambodian */
122 { "kn", NULL
, NULL
}, /* Kannada */
123 { "ko", NULL
, "EUC-KR"}, /* Korean */
124 { "ks", NULL
, NULL
}, /* Kashmiri */
125 { "ku", NULL
, NULL
}, /* Kurdish */
126 { "kv", NULL
, NULL
}, /* Komi */
127 { "kw", NULL
, "iso-8859-14"}, /* Cornish: 1,5,8 */
128 { "ky", NULL
, NULL
}, /* Kirghiz */
129 { "la", NULL
, "iso-8859-1"}, /* Latin */
130 { "lb", NULL
, "iso-8859-1"}, /* Letzeburgesch */
131 { "ln", NULL
, NULL
}, /* Lingala */
132 { "lo", NULL
, NULL
}, /* Lao; Laotian */
133 { "lt", NULL
, "iso-8859-4"}, /* Lithuanian */
134 { "lv", NULL
, "iso-8859-4"}, /* Latvian; Lettish */
135 { "mg", NULL
, NULL
}, /* Malagasy */
136 { "mh", NULL
, NULL
}, /* Marshall */
137 { "mi", NULL
, NULL
}, /* Maori */
138 { "mk", NULL
, NULL
}, /* Macedonian */
139 { "ml", NULL
, NULL
}, /* Malayalam */
140 { "mn", NULL
, NULL
}, /* Mongolian */
141 { "mo", NULL
, "iso-8859-2"}, /* Moldavian */
142 { "mr", NULL
, NULL
}, /* Marathi */
143 { "ms", NULL
, NULL
}, /* Malay */
144 { "mt", NULL
, "iso-8859-3"}, /* Maltese */
145 { "my", NULL
, NULL
}, /* Burmese */
146 { "na", NULL
, NULL
}, /* Nauru */
147 { "nb", NULL
, "iso-8859-1"}, /* Norwegian Bokmål; Bokm@aa{}l */
148 { "nd", NULL
, NULL
}, /* Ndebele, North */
149 { "ne", NULL
, NULL
}, /* Nepali */
150 { "ng", NULL
, NULL
}, /* Ndonga */
151 { "nl", NULL
, "iso-8859-1"}, /* Dutch: 5,9 */
152 { "nn", NULL
, "iso-8859-1"}, /* Norwegian Nynorsk */
153 { "no", NULL
, "iso-8859-1"}, /* Norwegian */
154 { "nr", NULL
, NULL
}, /* Ndebele, South */
155 { "nv", NULL
, NULL
}, /* Navajo */
156 { "ny", NULL
, NULL
}, /* Chichewa; Nyanja */
157 { "oc", NULL
, NULL
}, /* Occitan; Provençal; Proven@,{c}al */
158 { "om", NULL
, NULL
}, /* (Afan) Oromo */
159 { "or", NULL
, NULL
}, /* Oriya */
160 { "os", NULL
, NULL
}, /* Ossetian; Ossetic */
161 { "pa", NULL
, NULL
}, /* Panjabi; Punjabi */
162 { "pi", NULL
, NULL
}, /* Pali */
163 { "pl", NULL
, "iso-8859-2"}, /* Polish */
164 { "ps", NULL
, NULL
}, /* Pashto, Pushto */
165 { "pt", NULL
, "iso-8859-1"}, /* Portuguese */
166 { "qu", NULL
, "iso-8859-1"}, /* Quechua */
167 { "rm", NULL
, "iso-8859-1"}, /* Rhaeto-Romance */
168 { "rn", NULL
, NULL
}, /* Rundi; Kirundi */
169 { "ro", NULL
, "iso-8859-2"}, /* Romanian */
170 { "ru", NULL
, "koi8-r"}, /* Russian */
171 { "rw", NULL
, NULL
}, /* Kinyarwanda */
172 { "sa", NULL
, NULL
}, /* Sanskrit */
173 { "sc", NULL
, "iso-8859-1"}, /* Sardinian */
174 { "sd", NULL
, NULL
}, /* Sindhi */
175 { "se", NULL
, "iso-8859-10"}, /* Northern Sami */
176 { "sg", NULL
, NULL
}, /* Sango; Sangro */
177 { "si", NULL
, NULL
}, /* Sinhalese */
178 { "sk", NULL
, "iso-8859-2"}, /* Slovak */
179 { "sl", NULL
, "iso-8859-1"}, /* Slovenian */
180 { "sm", NULL
, NULL
}, /* Samoan */
181 { "sn", NULL
, NULL
}, /* Shona */
182 { "so", NULL
, NULL
}, /* Somali */
183 { "sq", NULL
, "iso-8859-1"}, /* Albanian: 2,5,8,9,10 */
184 { "sr", NULL
, "iso-8859-2"}, /* Serbian */
185 { "ss", NULL
, NULL
}, /* Swati; Siswati */
186 { "st", NULL
, NULL
}, /* Sesotho; Sotho, Southern */
187 { "su", NULL
, NULL
}, /* Sundanese */
188 { "sv", NULL
, "iso-8859-1"}, /* Swedish */
189 { "sw", NULL
, NULL
}, /* Swahili */
190 { "ta", NULL
, NULL
}, /* Tamil */
191 { "te", NULL
, NULL
}, /* Telugu */
192 { "tg", NULL
, NULL
}, /* Tajik */
193 { "th", NULL
, "iso-8859-11"}, /* Thai */
194 { "ti", NULL
, NULL
}, /* Tigrinya */
195 { "tk", NULL
, NULL
}, /* Turkmen */
196 { "tl", NULL
, "iso-8859-1"}, /* Tagalog */
197 { "tn", NULL
, NULL
}, /* Tswana; Setswana */
198 { "to", NULL
, NULL
}, /* Tonga (?) */
199 { "tr", NULL
, "iso-8859-9"}, /* Turkish */
200 { "ts", NULL
, NULL
}, /* Tsonga */
201 { "tt", NULL
, NULL
}, /* Tatar */
202 { "tw", NULL
, NULL
}, /* Twi */
203 { "ty", NULL
, NULL
}, /* Tahitian */
204 { "ug", NULL
, NULL
}, /* Uighur */
205 { "uk", NULL
, "koi8-u"}, /* Ukrainian */
206 { "ur", NULL
, NULL
}, /* Urdu */
207 { "uz", NULL
, NULL
}, /* Uzbek */
208 { "vi", NULL
, NULL
}, /* Vietnamese */
209 { "vo", NULL
, NULL
}, /* Volapük; Volap@"{u}k; Volapuk */
210 { "wa", NULL
, "iso-8859-1"}, /* Walloon */
211 { "wo", NULL
, NULL
}, /* Wolof */
212 { "xh", NULL
, NULL
}, /* Xhosa */
213 { "yi", NULL
, "iso-8859-8"}, /* Yiddish (formerly ji) */
214 { "yo", NULL
, NULL
}, /* Yoruba */
215 { "za", NULL
, NULL
}, /* Zhuang */
216 { "zh", "TW", "big5"}, /* Chinese */
217 { "zh", NULL
, "gb2312"}, /* Chinese */
218 { "zu", NULL
, NULL
}, /* Zulu */
222 /* Given the language and (optionally) territory code, return the
223 default character set for that language. See notes above. */
226 charset_lookup (char *lang
, char *terr
)
228 static struct langtab
*p
;
232 for (p
= langtab
; p
->lang
; p
++)
233 if (strcasecmp (p
->lang
, lang
) == 0
236 || !strcasecmp (p
->terr
, terr
) == 0))
244 const char *charset
= NULL
;
247 /* Try to deduce the charset from LC_ALL or LANG variables */
249 tmp
= getenv ("LC_ALL");
251 tmp
= getenv ("LANG");
258 lang
= strtok (tmp
, "_");
259 terr
= strtok (NULL
, ".");
260 charset
= strtok (NULL
, "@");
263 charset
= charset_lookup (lang
, terr
);
267 charset
= "iso-8859-1";
273 #ifndef HAVE_LIBICONV
276 iconv_open (const char *tocode
, const char *fromcode
)
278 return (iconv_t
)(-1);
282 iconv (iconv_t cd
, ICONV_CONST
char **inbuf
, size_t *inbytesleft
,
283 char **outbuf
, size_t *outbytesleft
)
289 iconv_close (iconv_t cd
)
294 #endif /* !HAVE_LIBICONV */
299 static iconv_t conv_desc
[2] = { (iconv_t
) -1, (iconv_t
) -1 };
302 utf8_init (bool to_utf
)
304 if (conv_desc
[(int) to_utf
] == (iconv_t
) -1)
307 conv_desc
[(int) to_utf
] = iconv_open ("UTF-8", get_input_charset ());
309 conv_desc
[(int) to_utf
] = iconv_open (get_input_charset (), "UTF-8");
311 return conv_desc
[(int) to_utf
];
315 utf8_convert(bool to_utf
, const char *input
, char **output
)
322 iconv_t cd
= utf8_init (to_utf
);
326 *output
= xstrdup (input
);
329 else if (cd
== (iconv_t
)-1)
332 inlen
= strlen (input
) + 1;
333 outlen
= inlen
* MB_LEN_MAX
+ 1;
334 ob
= *output
= xmalloc (outlen
);
336 rc
= iconv (cd
, &ib
, &inlen
, &ob
, &outlen
);
343 string_ascii_p (const char *str
)
345 const unsigned char *p
= (const unsigned char *)str
;
This page took 0.053467 seconds and 4 git commands to generate.