2 * This program is free software; you can redistribute it and/or modify
3 * it under the terms of the GNU General Public License as published by
4 * the Free Software Foundation; either version 2 of the License, or (at
5 * your option) any later version.
7 * This program is distributed in the hope that it will be useful, but
8 * WITHOUT ANY WARRANTY; without even the implied warranty of
9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
10 * General Public License for more details.
12 * You should have received a copy of the GNU General Public License
13 * along with this program; if not, write to the Free Software
14 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
17 * Copyright © 2006 Pierre Habouzit
20 * Copyright notice from original mutt:
21 * Copyright (C) 1999-2000 Thomas Roessler <roessler@does-not-exist.org>
23 * This file is part of mutt-ng, see http://www.muttng.org/.
24 * It's licensed under the GNU General Public License,
25 * please see the file GPL in the top level source directory.
28 #include <lib-lib/lib-lib.h>
30 #ifdef HAVE_LANGINFO_CODESET
31 # include <langinfo.h>
37 # define EILSEQ EINVAL
39 @import "lib-lua/base.cpkg"
44 ** This variable is a colon-separated list of character encoding
45 ** schemes for messages without character encoding indication.
46 ** Header field values and message body content without character encoding
47 ** indication would be assumed that they are written in one of this list.
48 ** By default, all the header fields and message body without any charset
49 ** indication are assumed to be in \fTus-ascii\fP.
51 ** For example, Japanese users might prefer this:
53 ** \fTset assumed_charset="iso-2022-jp:euc-jp:shift_jis:utf-8"\fP
55 ** However, only the first content is valid for the message body.
56 ** This variable is valid only if $$strict_mime is unset.
58 string_t assumed_charset = m_strdup("us-ascii");
62 ** Character set your terminal uses to display and enter textual data.
64 string_t charset = NULL;
68 ** This variable is a colon-separated list of character encoding
69 ** schemes for text file attatchments.
70 ** If \fIunset\fP, $$charset value will be used instead.
71 ** For example, the following configuration would work for Japanese
74 ** \fTset file_charset="iso-2022-jp:euc-jp:shift_jis:utf-8"\fP
76 ** Note: ``\fTiso-2022-*\fP'' must be put at the head of the value as shown above
79 string_t file_charset = NULL;
83 ** A list of character sets for outgoing messages. Madmutt will use the
84 ** first character set into which the text can be converted exactly.
85 ** If your ``$$charset'' is not \fTiso-8859-1\fP and recipients may not
86 ** understand \fTUTF-8\fP, it is advisable to include in the list an
87 ** appropriate widely used standard character set (such as
88 ** \fTiso-8859-2\fP, \fTkoi8-r\fP or \fTiso-2022-jp\fP) either
89 ** instead of or after \fTiso-8859-1\fP.
91 string_t send_charset = m_strdup("us-ascii:iso-8859-1:utf-8");
94 int Charset_is_utf8 = 0;
95 wchar_t CharsetReplacement = '?';
98 /****************************************************************************/
99 /* charset functions */
100 /****************************************************************************/
102 void charset_initialize(void)
104 #ifdef HAVE_LANGINFO_CODESET
108 m_strcpy(buff, sizeof(buff), nl_langinfo(CODESET));
109 charset_canonicalize(buff2, sizeof(buff2), buff);
111 /* finally, set $charset */
112 if (!m_strisempty(buff2)) {
113 m_strreplace(&MCharset.charset, buff2);
117 m_strreplace(&MCharset.charset, "iso-8859-1");
120 Charset_is_utf8 = !m_strcmp(MCharset.charset, "utf-8");
121 CharsetReplacement = Charset_is_utf8 ? 0xfffd : '?';
123 #ifdef HAVE_BIND_TEXTDOMAIN_CODESET
124 bind_textdomain_codeset(PACKAGE, MCharset.charset);
128 #include "charset.gperf"
129 void charset_canonicalize(char *dest, ssize_t dlen, const char *name)
131 const struct cset_pair *cp;
132 char scratch[STRING];
137 m_strcpy(dest, dlen, "us-ascii");
141 // canonize name: only keep a-z0-9 and dots, put into lowercase
142 for (p = name; *p && *p != ':' && i < ssizeof(scratch) - 1; p++) {
143 if (isalnum(*p) || *p== '.') {
144 scratch[i++] = tolower((unsigned char)*p);
149 cp = charset_canonicalize_aux(scratch, strlen(scratch));
151 m_strcpy(dest, dlen, cp->pref);
153 m_strcpy(dest, dlen, name);
158 /* XXX: MC: UGLY return of local static */
159 const char *charset_getfirst(const char *charset)
161 static char fcharset[STRING];
164 if (m_strisempty(charset))
167 p = m_strchrnul(charset, ':');
168 m_strncpy(fcharset, sizeof(fcharset), charset, p - charset);
172 int charset_is_utf8(const char *s)
175 charset_canonicalize(buf, sizeof(buf), s);
176 return !m_strcmp(buf, "utf-8");
179 int charset_is_us_ascii(const char *s)
182 charset_canonicalize(buf, sizeof(buf), s);
183 return !m_strcmp(buf, "us-ascii");
187 /****************************************************************************/
188 /* iconv-line functions */
189 /****************************************************************************/
191 /* Like iconv_open, but canonicalises the charsets */
192 iconv_t mutt_iconv_open(const char *tocode, const char *fromcode, int flags)
194 char tocode1[STRING];
195 char fromcode1[STRING];
200 if ((flags & M_ICONV_HOOK_TO) && (tmp = mutt_charset_hook(tocode1))) {
201 charset_canonicalize(tocode1, sizeof(tocode1), tmp);
203 charset_canonicalize(tocode1, sizeof(tocode1), tocode);
206 if ((flags & M_ICONV_HOOK_FROM) && (tmp = mutt_charset_hook(fromcode1))) {
207 charset_canonicalize(fromcode1, sizeof(fromcode1), tmp);
209 charset_canonicalize(fromcode1, sizeof(fromcode1), fromcode);
212 cd = iconv_open(tocode1, fromcode1);
213 if (cd != MUTT_ICONV_ERROR)
217 const char *to = mutt_iconv_hook(tocode1);
218 const char *from = mutt_iconv_hook(fromcode1);
220 return to && from ? iconv_open(to, from) : MUTT_ICONV_ERROR;
225 /* Like iconv, but keeps going even when the input is invalid
226 If you're supplying inrepls, the source charset should be stateless;
227 if you're supplying an outrepl, the target charset should be. */
228 /* XXX: MC: I do not understand what it does yet */
229 ssize_t mutt_iconv(iconv_t cd,
230 const char **inbuf, ssize_t *inbytesleft,
231 char **outbuf, ssize_t *outbytesleft,
232 const char **inrepls, const char *outrepl)
234 ssize_t ret = 0, ret1;
235 const char *ib = *inbuf;
236 ssize_t ibl = *inbytesleft;
238 ssize_t obl = *outbytesleft;
241 ret1 = my_iconv(cd, &ib, &ibl, &ob, &obl);
245 if (ibl && obl && errno == EILSEQ) {
247 /* Try replacing the input */
250 for (t = inrepls; *t; t++) {
251 const char *ib1 = *t;
252 ssize_t ibl1 = m_strlen(*t);
256 my_iconv(cd, &ib1, &ibl1, &ob1, &obl1);
259 ob = ob1, obl = obl1;
267 /* Replace the output */
270 my_iconv(cd, 0, 0, &ob, &obl);
272 ssize_t n = m_strlen(outrepl);
278 memcpy(ob, outrepl, n);
282 my_iconv(cd, 0, 0, 0, 0); /* for good measure */
286 *inbuf = ib, *inbytesleft = ibl;
287 *outbuf = ob, *outbytesleft = obl;
292 /* Convert a string */
294 mutt_convert_string(char **ps, const char *from, const char *to, int flags)
297 const char *repls[] = { "\357\277\275", "?", 0 };
299 if (m_strisempty(*ps))
302 cd = mutt_iconv_open(to, from, flags);
303 if (cd != MUTT_ICONV_ERROR) {
304 const char **inrepls = NULL;
305 const char *outrepl = NULL;
310 if (charset_is_utf8(to))
311 outrepl = "\357\277\275";
313 if (charset_is_utf8(from))
318 ibl = m_strlen(*ps) + 1;
321 obl = MB_LEN_MAX * ibl;
322 ob = buf = p_new(char, obl + 1);
324 mutt_iconv(cd, &ib, &ibl, &ob, &obl, inrepls, outrepl);
337 static ssize_t convert_string(const char *f, ssize_t flen,
338 const char *from, const char *to,
339 char **t, ssize_t * tlen)
347 if ((cd = mutt_iconv_open(to, from, 0)) == MUTT_ICONV_ERROR)
351 ob = buf = p_new(char, obl);
352 n = my_iconv(cd, &f, &flen, &ob, &obl);
354 if (n < 0 || my_iconv(cd, 0, 0, &ob, &obl) < 0) {
369 int mutt_convert_nonmime_string(char **ps)
371 const char *p = MCharset.assumed_charset;
372 ssize_t ulen = m_strlen(*ps);
377 char fromcode[LONG_STRING], *s = NULL;
386 q = m_strchrnul(p, ':');
387 m_strncpy(fromcode, sizeof(fromcode), p, q - p);
390 if (convert_string(u, ulen, fromcode, MCharset.charset, &s, &slen) >= 0) {
400 /****************************************************************************/
401 /* fgetconv functions */
402 /****************************************************************************/
404 /* fgetconv_t stuff for converting a file while reading it
405 Used in sendlib.c for converting from mutt's charset */
416 const char **inrepls;
420 fgetconv_open(FILE *file, const char *from, const char *to, int flags)
422 static const char *repls[] = { "\357\277\275", "?", 0 };
424 struct fgetconv_t *fc = p_new(struct fgetconv_t, 1);
427 fc->cd = MUTT_ICONV_ERROR;
429 fc->cd = mutt_iconv_open(to, from, flags);
431 if (fc->cd != MUTT_ICONV_ERROR) {
432 fc->p = fc->ob = fc->bufo;
435 fc->inrepls = repls + charset_is_utf8(to);
440 void fgetconv_close(fgetconv_t **fcp)
442 struct fgetconv_t *fc = *fcp;
444 if (fc->cd != MUTT_ICONV_ERROR)
445 iconv_close (fc->cd);
450 int fgetconv(fgetconv_t *fc)
455 if (fc->cd == MUTT_ICONV_ERROR)
456 return fgetc(fc->file);
461 return (unsigned char)*(fc->p)++;
463 /* Try to convert some more */
464 fc->p = fc->ob = fc->bufo;
466 ssize_t obl = ssizeof(fc->bufo);
468 my_iconv(fc->cd, (const char **)&fc->ib, &fc->ibl, &fc->ob, &obl);
470 return (unsigned char)*(fc->p)++;
473 /* If we trusted iconv a bit more, we would at this point
474 * ask why it had stopped converting ... */
476 /* Try to read some more */
477 if (fc->ibl == sizeof(fc->bufi)
478 || (fc->ibl && fc->ib + fc->ibl < fc->bufi + sizeof(fc->bufi))) {
484 memcpy(fc->bufi, fc->ib, fc->ibl);
487 fc->ibl += fread(fc->ib + fc->ibl, 1, sizeof(fc->bufi) - fc->ibl,
490 /* Try harder this time to convert some */
492 ssize_t obl = ssizeof(fc->bufo);
494 mutt_iconv(fc->cd, (const char **)&fc->ib, &fc->ibl, &fc->ob, &obl,
496 if (fc->p < fc->ob) {
497 return (unsigned char)*(fc->p)++;
501 /* Either the file has finished or one of the buffers is too small */
506 char *fgetconvs(char *buf, ssize_t len, fgetconv_t *fc)
510 while (pos < len - 1) {
511 int c = fgetconv(fc);
521 return pos ? buf : NULL;