2 * Copyright (C) 1999-2000 Thomas Roessler <roessler@does-not-exist.org>
4 * This program is free software; you can redistribute it
5 * and/or modify it under the terms of the GNU General Public
6 * License as published by the Free Software Foundation; either
7 * version 2 of the License, or (at your option) any later
10 * This program is distributed in the hope that it will be
11 * useful, but WITHOUT ANY WARRANTY; without even the implied
12 * warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
13 * PURPOSE. See the GNU General Public License for more
16 * You should have received a copy of the GNU General Public
17 * License along with this program; if not, write to the Free
18 * Software Foundation, Inc., 59 Temple Place - Suite 330,
19 * Boston, MA 02111, USA.
32 #include <sys/types.h>
41 # define EILSEQ EINVAL
45 * The following list has been created manually from the data under:
46 * http://www.isi.edu/in-notes/iana/assignments/character-sets
47 * Last update: 2000-09-07
49 * Note that it includes only the subset of character sets for which
50 * a preferred MIME name is given.
56 } PreferredMIMENames[] = {
58 "ansi_x3.4-1968", "us-ascii"}, {
59 "iso-ir-6", "us-ascii"}, {
60 "iso_646.irv:1991", "us-ascii"}, {
61 "ascii", "us-ascii"}, {
62 "iso646-us", "us-ascii"}, {
64 "ibm367", "us-ascii"}, {
65 "cp367", "us-ascii"}, {
66 "csASCII", "us-ascii"}, {
67 "csISO2022KR", "iso-2022-kr"}, {
68 "csEUCKR", "euc-kr"}, {
69 "csISO2022JP", "iso-2022-jp"}, {
70 "csISO2022JP2", "iso-2022-jp-2"}, {
71 "ISO_8859-1:1987", "iso-8859-1"}, {
72 "iso-ir-100", "iso-8859-1"}, {
73 "iso_8859-1", "iso-8859-1"}, {
74 "latin1", "iso-8859-1"}, {
75 "l1", "iso-8859-1"}, {
76 "IBM819", "iso-8859-1"}, {
77 "CP819", "iso-8859-1"}, {
78 "csISOLatin1", "iso-8859-1"}, {
79 "ISO_8859-2:1987", "iso-8859-2"}, {
80 "iso-ir-101", "iso-8859-2"}, {
81 "iso_8859-2", "iso-8859-2"}, {
82 "latin2", "iso-8859-2"}, {
83 "l2", "iso-8859-2"}, {
84 "csISOLatin2", "iso-8859-2"}, {
85 "ISO_8859-3:1988", "iso-8859-3"}, {
86 "iso-ir-109", "iso-8859-3"}, {
87 "ISO_8859-3", "iso-8859-3"}, {
88 "latin3", "iso-8859-3"}, {
89 "l3", "iso-8859-3"}, {
90 "csISOLatin3", "iso-8859-3"}, {
91 "ISO_8859-4:1988", "iso-8859-4"}, {
92 "iso-ir-110", "iso-8859-4"}, {
93 "ISO_8859-4", "iso-8859-4"}, {
94 "latin4", "iso-8859-4"}, {
95 "l4", "iso-8859-4"}, {
96 "csISOLatin4", "iso-8859-4"}, {
97 "ISO_8859-6:1987", "iso-8859-6"}, {
98 "iso-ir-127", "iso-8859-6"}, {
99 "iso_8859-6", "iso-8859-6"}, {
100 "ECMA-114", "iso-8859-6"}, {
101 "ASMO-708", "iso-8859-6"}, {
102 "arabic", "iso-8859-6"}, {
103 "csISOLatinArabic", "iso-8859-6"}, {
104 "ISO_8859-7:1987", "iso-8859-7"}, {
105 "iso-ir-126", "iso-8859-7"}, {
106 "ISO_8859-7", "iso-8859-7"}, {
107 "ELOT_928", "iso-8859-7"}, {
108 "ECMA-118", "iso-8859-7"}, {
109 "greek", "iso-8859-7"}, {
110 "greek8", "iso-8859-7"}, {
111 "csISOLatinGreek", "iso-8859-7"}, {
112 "ISO_8859-8:1988", "iso-8859-8"}, {
113 "iso-ir-138", "iso-8859-8"}, {
114 "ISO_8859-8", "iso-8859-8"}, {
115 "hebrew", "iso-8859-8"}, {
116 "csISOLatinHebrew", "iso-8859-8"}, {
117 "ISO_8859-5:1988", "iso-8859-5"}, {
118 "iso-ir-144", "iso-8859-5"}, {
119 "ISO_8859-5", "iso-8859-5"}, {
120 "cyrillic", "iso-8859-5"}, {
121 "csISOLatinCyrillic", "iso8859-5"}, {
122 "ISO_8859-9:1989", "iso-8859-9"}, {
123 "iso-ir-148", "iso-8859-9"}, {
124 "ISO_8859-9", "iso-8859-9"}, {
125 "latin5", "iso-8859-9"}, /* this is not a bug */
127 "l5", "iso-8859-9"}, {
128 "csISOLatin5", "iso-8859-9"}, {
129 "ISO_8859-10:1992", "iso-8859-10"}, {
130 "iso-ir-157", "iso-8859-10"}, {
131 "latin6", "iso-8859-10"}, /* this is not a bug */
133 "l6", "iso-8859-10"}, {
134 "csISOLatin6" "iso-8859-10"}, {
135 "csKOI8r", "koi8-r"}, {
136 "MS_Kanji", "Shift_JIS"}, /* Note the underscore! */
138 "csShiftJis", "Shift_JIS"}, {
139 "Extended_UNIX_Code_Packed_Format_for_Japanese", "EUC-JP"}, {
140 "csEUCPkdFmtJapanese", "EUC-JP"}, {
141 "csGB2312", "gb2312"}, {
144 * End of official brain damage. What follows has been taken
145 * from glibc's localedata files.
148 "iso_8859-13", "iso-8859-13"}, {
149 "iso-ir-179", "iso-8859-13"}, {
150 "latin7", "iso-8859-13"}, /* this is not a bug */
152 "l7", "iso-8859-13"}, {
153 "iso_8859-14", "iso-8859-14"}, {
154 "latin8", "iso-8859-14"}, /* this is not a bug */
156 "l8", "iso-8859-14"}, {
157 "iso_8859-15", "iso-8859-15"}, {
158 "latin9", "iso-8859-15"}, /* this is not a bug */
159 /* Suggested by Ionel Mugurel Ciobica <tgakic@sg10.chem.tue.nl> */
161 "latin0", "iso-8859-15"}, /* this is not a bug */
163 "iso_8859-16", "iso-8859-16"}, {
164 "latin10", "iso-8859-16"}, /* this is not a bug */
166 * David Champion <dgc@uchicago.edu> has observed this with
167 * nl_langinfo under SunOS 5.8.
172 * http://www.sun.com/software/white-papers/wp-unicode/
175 "eucJP", "euc-jp"}, {
176 "PCK", "Shift_JIS"}, {
177 "ko_KR-euc", "euc-kr"}, {
178 "zh_TW-big5", "big5"},
179 /* seems to be common on some systems */
181 "sjis", "Shift_JIS"}, {
182 "euc-jp-ms", "eucJP-ms"},
184 * If you happen to encounter system-specific brain-damage with
185 * respect to character set naming, please add it above this
186 * comment, and submit a patch to <mutt-dev@mutt.org>.
188 /* End of aliases. Please keep this line last. */
193 #ifdef HAVE_LANGINFO_CODESET
194 # include <langinfo.h>
197 void mutt_set_langinfo_charset (void)
199 char buff[LONG_STRING];
200 char buff2[LONG_STRING];
202 strfcpy (buff, nl_langinfo (CODESET), sizeof (buff));
203 mutt_canonical_charset (buff2, sizeof (buff2), buff);
205 /* finally, set $charset */
206 if (!(Charset = safe_strdup (buff2)))
207 Charset = safe_strdup ("iso-8859-1");
212 void mutt_set_langinfo_charset (void)
214 Charset = safe_strdup ("iso-8859-1");
219 void mutt_canonical_charset (char *dest, size_t dlen, const char *name)
223 char scratch[LONG_STRING];
225 /* catch some common iso-8859-something misspellings */
226 if (!ascii_strncasecmp (name, "8859", 4) && name[4] != '-')
227 snprintf (scratch, sizeof (scratch), "iso-8859-%s", name + 4);
228 else if (!ascii_strncasecmp (name, "8859-", 5))
229 snprintf (scratch, sizeof (scratch), "iso-8859-%s", name + 5);
230 else if (!ascii_strncasecmp (name, "iso8859", 7) && name[7] != '-')
231 snprintf (scratch, sizeof (scratch), "iso_8859-%s", name + 7);
232 else if (!ascii_strncasecmp (name, "iso8859-", 8))
233 snprintf (scratch, sizeof (scratch), "iso_8859-%s", name + 8);
235 strfcpy (scratch, NONULL (name), sizeof (scratch));
237 for (i = 0; PreferredMIMENames[i].key; i++)
238 if (!ascii_strcasecmp (scratch, PreferredMIMENames[i].key) ||
239 !mutt_strcasecmp (scratch, PreferredMIMENames[i].key)) {
240 strfcpy (dest, PreferredMIMENames[i].pref, dlen);
244 strfcpy (dest, scratch, dlen);
246 /* for cosmetics' sake, transform to lowercase. */
247 for (p = dest; *p; p++)
248 *p = ascii_tolower (*p);
251 int mutt_chscmp (const char *s, const char *chs)
258 mutt_canonical_charset (buffer, sizeof (buffer), s);
259 return !ascii_strcasecmp (buffer, chs);
265 iconv_t iconv_open (const char *tocode, const char *fromcode)
267 return (iconv_t) (-1);
270 size_t iconv (iconv_t cd, ICONV_CONST char **inbuf, size_t * inbytesleft,
271 char **outbuf, size_t * outbytesleft)
276 int iconv_close (iconv_t cd)
281 #endif /* !HAVE_ICONV */
285 * Like iconv_open, but canonicalises the charsets
288 iconv_t mutt_iconv_open (const char *tocode, const char *fromcode, int flags)
290 char tocode1[SHORT_STRING];
291 char fromcode1[SHORT_STRING];
292 char *tocode2, *fromcode2;
297 mutt_canonical_charset (tocode1, sizeof (tocode1), tocode);
299 #ifdef M_ICONV_HOOK_TO
301 if ((flags & M_ICONV_HOOK_TO) && (tmp = mutt_charset_hook (tocode1)))
302 mutt_canonical_charset (tocode1, sizeof (tocode1), tmp);
305 mutt_canonical_charset (fromcode1, sizeof (fromcode1), fromcode);
306 if ((flags & M_ICONV_HOOK_FROM) && (tmp = mutt_charset_hook (fromcode1)))
307 mutt_canonical_charset (fromcode1, sizeof (fromcode1), tmp);
309 if ((cd = iconv_open (tocode1, fromcode1)) != (iconv_t) - 1)
311 if ((tocode2 = mutt_iconv_hook (tocode1))
312 && (fromcode2 = mutt_iconv_hook (fromcode1)))
313 return iconv_open (tocode2, fromcode2);
315 return (iconv_t) - 1;
320 * Like iconv, but keeps going even when the input is invalid
321 * If you're supplying inrepls, the source charset should be stateless;
322 * if you're supplying an outrepl, the target charset should be.
325 size_t mutt_iconv (iconv_t cd, ICONV_CONST char **inbuf, size_t * inbytesleft,
326 char **outbuf, size_t * outbytesleft,
327 ICONV_CONST char **inrepls, const char *outrepl)
329 size_t ret = 0, ret1;
330 ICONV_CONST char *ib = *inbuf;
331 size_t ibl = *inbytesleft;
333 size_t obl = *outbytesleft;
336 ret1 = iconv (cd, &ib, &ibl, &ob, &obl);
337 if (ret1 != (size_t) - 1)
339 if (ibl && obl && errno == EILSEQ) {
341 /* Try replacing the input */
342 ICONV_CONST char **t;
344 for (t = inrepls; *t; t++) {
345 ICONV_CONST char *ib1 = *t;
346 size_t ibl1 = strlen (*t);
350 iconv (cd, &ib1, &ibl1, &ob1, &obl1);
353 ob = ob1, obl = obl1;
361 /* Replace the output */
364 iconv (cd, 0, 0, &ob, &obl);
366 int n = strlen (outrepl);
372 memcpy (ob, outrepl, n);
376 iconv (cd, 0, 0, 0, 0); /* for good measure */
380 *inbuf = ib, *inbytesleft = ibl;
381 *outbuf = ob, *outbytesleft = obl;
389 * Used in rfc2047.c and rfc2231.c
392 int mutt_convert_string (char **ps, const char *from, const char *to,
396 ICONV_CONST char *repls[] = { "\357\277\275", "?", 0 };
402 if (to && from && (cd = mutt_iconv_open (to, from, flags)) != (iconv_t) - 1) {
404 ICONV_CONST char *ib;
407 ICONV_CONST char **inrepls = 0;
410 if (mutt_is_utf8 (to))
411 outrepl = "\357\277\275";
412 else if (mutt_is_utf8 (from))
418 ib = s, ibl = len + 1;
419 obl = MB_LEN_MAX * ibl;
420 ob = buf = safe_malloc (obl + 1);
422 mutt_iconv (cd, &ib, &ibl, &ob, &obl, inrepls, outrepl);
430 mutt_str_adjust (ps);
439 * FGETCONV stuff for converting a file while reading it
440 * Used in sendlib.c for converting from mutt's Charset
452 ICONV_CONST char **inrepls;
455 struct fgetconv_not {
460 FGETCONV *fgetconv_open (FILE * file, const char *from, const char *to,
463 struct fgetconv_s *fc;
464 iconv_t cd = (iconv_t) - 1;
465 static ICONV_CONST char *repls[] = { "\357\277\275", "?", 0 };
468 cd = mutt_iconv_open (to, from, flags);
470 if (cd != (iconv_t) - 1) {
471 fc = safe_malloc (sizeof (struct fgetconv_s));
472 fc->p = fc->ob = fc->bufo;
475 fc->inrepls = mutt_is_utf8 (to) ? repls : repls + 1;
478 fc = safe_malloc (sizeof (struct fgetconv_not));
481 return (FGETCONV *) fc;
484 char *fgetconvs (char *buf, size_t l, FGETCONV * _fc)
489 for (r = 0; r + 1 < l;) {
490 if ((c = fgetconv (_fc)) == EOF)
504 int fgetconv (FGETCONV * _fc)
506 struct fgetconv_s *fc = (struct fgetconv_s *) _fc;
510 if (fc->cd == (iconv_t) - 1)
511 return fgetc (fc->file);
515 return (unsigned char) *(fc->p)++;
517 /* Try to convert some more */
518 fc->p = fc->ob = fc->bufo;
520 size_t obl = sizeof (fc->bufo);
522 iconv (fc->cd, (ICONV_CONST char **) &fc->ib, &fc->ibl, &fc->ob, &obl);
524 return (unsigned char) *(fc->p)++;
527 /* If we trusted iconv a bit more, we would at this point
528 * ask why it had stopped converting ... */
530 /* Try to read some more */
531 if (fc->ibl == sizeof (fc->bufi) ||
532 (fc->ibl && fc->ib + fc->ibl < fc->bufi + sizeof (fc->bufi))) {
537 memcpy (fc->bufi, fc->ib, fc->ibl);
540 fread (fc->ib + fc->ibl, 1, sizeof (fc->bufi) - fc->ibl, fc->file);
542 /* Try harder this time to convert some */
544 size_t obl = sizeof (fc->bufo);
546 mutt_iconv (fc->cd, (ICONV_CONST char **) &fc->ib, &fc->ibl, &fc->ob,
547 &obl, fc->inrepls, 0);
549 return (unsigned char) *(fc->p)++;
552 /* Either the file has finished or one of the buffers is too small */
557 void fgetconv_close (FGETCONV ** _fc)
559 struct fgetconv_s *fc = (struct fgetconv_s *) *_fc;
561 if (fc->cd != (iconv_t) - 1)
562 iconv_close (fc->cd);
566 char *mutt_get_first_charset (const char *charset)
568 static char fcharset[SHORT_STRING];
572 if (!mutt_strlen (c))
574 if (!(c1 = strchr (c, ':')))
576 strfcpy (fcharset, c, c1 - c + 1);
580 static size_t convert_string (ICONV_CONST char *f, size_t flen,
581 const char *from, const char *to,
582 char **t, size_t * tlen)
589 cd = mutt_iconv_open (to, from, 0);
590 if (cd == (iconv_t) (-1))
591 return (size_t) (-1);
593 ob = buf = safe_malloc (obl);
594 n = iconv (cd, &f, &flen, &ob, &obl);
595 if (n == (size_t) (-1) || iconv (cd, 0, 0, &ob, &obl) == (size_t) (-1)) {
600 return (size_t) (-1);
606 safe_realloc (&buf, ob - buf + 1);
613 int mutt_convert_nonmime_string (char **ps)
617 for (c = AssumedCharset; c; c = c1 ? c1 + 1 : 0) {
622 size_t ulen = mutt_strlen (*ps);
628 c1 = strchr (c, ':');
629 n = c1 ? c1 - c : mutt_strlen (c);
632 fromcode = safe_malloc (n + 1);
633 strfcpy (fromcode, c, n + 1);
634 m = convert_string (u, ulen, fromcode, Charset, &s, &slen);
636 if (m != (size_t) (-1)) {