/*
* Copyright (c) 1987, Fujitsu LTD. (Itaru ICHIKAWA).
- * Copyright (c) 1996-2009, The nkf Project.
+ * Copyright (c) 1996-2010, The nkf Project.
*
* This software is provided 'as-is', without any express or implied
* warranty. In no event will the authors be held liable for any damages
*
* 3. This notice may not be removed or altered from any source distribution.
*/
-#define NKF_VERSION "2.1.1"
-#define NKF_RELEASE_DATE "2010-01-25"
+#define NKF_VERSION "2.1.3"
+#define NKF_RELEASE_DATE "2012-03-17"
#define COPY_RIGHT \
"Copyright (C) 1987, FUJITSU LTD. (I.Ichikawa).\n" \
- "Copyright (C) 1996-2010, The nkf Project."
+ "Copyright (C) 1996-2012, The nkf Project."
#include "config.h"
#include "nkf.h"
{"ISO-2022-JP-2004", ISO_2022_JP_2004},
{"SHIFT_JIS", SHIFT_JIS},
{"SJIS", SHIFT_JIS},
+ {"MS_Kanji", SHIFT_JIS},
{"PCK", SHIFT_JIS},
{"WINDOWS-31J", WINDOWS_31J},
{"CSWINDOWS31J", WINDOWS_31J},
/* ignore ZERO WIDTH NO-BREAK SPACE */
static int no_best_fit_chars_f = FALSE;
static int input_endian = ENDIAN_BIG;
+static int input_bom_f = FALSE;
static nkf_char unicode_subchar = '?'; /* the regular substitution character */
static void (*encode_fallback)(nkf_char c) = NULL;
static void w_status(struct input_code *, nkf_char);
static unsigned char stdobuf[IOBUF_SIZE];
#endif
+#define NKF_UNSPECIFIED (-TRUE)
+
/* flags */
static int unbuf_f = FALSE;
static int estab_f = FALSE;
static int broken_f = FALSE; /* convert ESC-less broken JIS */
static int iso8859_f = FALSE; /* ISO8859 through */
static int mimeout_f = FALSE; /* base64 mode */
-static int x0201_f = X0201_DEFAULT; /* convert JIS X 0201 */
+static int x0201_f = NKF_UNSPECIFIED; /* convert JIS X 0201 */
static int iso2022jp_f = FALSE; /* replace non ISO-2022-JP with GETA */
#ifdef UNICODE_NORMALIZATION
{"UTF-16", 0, 0, 0, {0, 0, 0}, NULL, w_iconv16, 0},
{"UTF-32", 0, 0, 0, {0, 0, 0}, NULL, w_iconv32, 0},
#endif
- {0}
+ {NULL, 0, 0, 0, {0, 0, 0}, NULL, NULL, 0}
};
static int mimeout_mode = 0; /* 0, -1, 'Q', 'B', 1, 2 */
nkf_buf_new(int length)
{
nkf_buf_t *buf = nkf_xmalloc(sizeof(nkf_buf_t));
- buf->ptr = nkf_xmalloc(length);
+ buf->ptr = nkf_xmalloc(sizeof(nkf_char) * length);
buf->capa = length;
buf->len = 0;
return buf;
-}
+}
#if 0
static void
" Z[0-4] Default/0: Convert JISX0208 Alphabet to ASCII\n"
" 1: Kankaku to one space 2: to two spaces 3: HTML Entity\n"
" 4: JISX0208 Katakana to JISX0201 Katakana\n"
- " X,x Assume X0201 kana in MS-Kanji, -x preserves X0201\n"
+ " X,x Convert Halfwidth Katakana to Fullwidth or preserve it\n"
);
fprintf(HELP_OUTPUT,
" O Output to File (DEFAULT 'nkf.out')\n"
case ISO_8859_1:
iso8859_f = TRUE;
break;
- case CP50220:
case CP50221:
case CP50222:
- x0201_f = TRUE;
+ if (x0201_f == NKF_UNSPECIFIED) x0201_f = FALSE; /* -x specified implicitly */
+ case CP50220:
#ifdef SHIFTJIS_CP932
cp51932_f = TRUE;
#endif
case SHIFT_JIS:
break;
case WINDOWS_31J:
- x0201_f = TRUE;
+ if (x0201_f == NKF_UNSPECIFIED) x0201_f = FALSE; /* -x specified implicitly */
#ifdef SHIFTJIS_CP932
cp51932_f = TRUE;
#endif
case EUCJP_NKF:
break;
case CP51932:
- x0201_f = TRUE;
+ if (x0201_f == NKF_UNSPECIFIED) x0201_f = FALSE; /* -x specified implicitly */
#ifdef SHIFTJIS_CP932
cp51932_f = TRUE;
#endif
#endif
break;
case EUCJP_MS:
+ if (x0201_f == NKF_UNSPECIFIED) x0201_f = FALSE; /* -x specified implicitly */
#ifdef SHIFTJIS_CP932
cp51932_f = FALSE;
#endif
#endif
break;
case EUCJP_ASCII:
+ if (x0201_f == NKF_UNSPECIFIED) x0201_f = FALSE; /* -x specified implicitly */
#ifdef SHIFTJIS_CP932
cp51932_f = FALSE;
#endif
{
switch (nkf_enc_to_index(enc)) {
case CP50220:
- x0201_f = TRUE;
#ifdef SHIFTJIS_CP932
if (cp932inv_f == TRUE) cp932inv_f = FALSE;
#endif
#endif
break;
case CP50221:
- x0201_f = TRUE;
+ if (x0201_f == NKF_UNSPECIFIED) x0201_f = FALSE; /* -x specified implicitly */
#ifdef SHIFTJIS_CP932
if (cp932inv_f == TRUE) cp932inv_f = FALSE;
#endif
case SHIFT_JIS:
break;
case WINDOWS_31J:
- x0201_f = TRUE;
+ if (x0201_f == NKF_UNSPECIFIED) x0201_f = FALSE; /* -x specified implicitly */
#ifdef UTF8_OUTPUT_ENABLE
ms_ucs_map_f = UCS_MAP_CP932;
#endif
#endif
break;
case CP51932:
- x0201_f = TRUE;
+ if (x0201_f == NKF_UNSPECIFIED) x0201_f = FALSE; /* -x specified implicitly */
#ifdef SHIFTJIS_CP932
if (cp932inv_f == TRUE) cp932inv_f = FALSE;
#endif
#endif
break;
case EUCJP_MS:
+ if (x0201_f == NKF_UNSPECIFIED) x0201_f = FALSE; /* -x specified implicitly */
x0212_f = TRUE;
#ifdef UTF8_OUTPUT_ENABLE
ms_ucs_map_f = UCS_MAP_MS;
#endif
break;
case EUCJP_ASCII:
+ if (x0201_f == NKF_UNSPECIFIED) x0201_f = FALSE; /* -x specified implicitly */
x0212_f = TRUE;
#ifdef UTF8_OUTPUT_ENABLE
ms_ucs_map_f = UCS_MAP_ASCII;
return 0;
}
-#define NKF_ICONV_NEED_ONE_MORE_BYTE -1
-#define NKF_ICONV_NEED_TWO_MORE_BYTES -2
+#define NKF_ICONV_NEED_ONE_MORE_BYTE (size_t)-1
+#define NKF_ICONV_NEED_TWO_MORE_BYTES (size_t)-2
#define UTF16_TO_UTF32(lead, trail) (((lead) << 10) + (trail) - NKF_INT32_C(0x35FDC00))
static size_t
nkf_iconv_utf_16(nkf_char c1, nkf_char c2, nkf_char c3, nkf_char c4)
}
#endif /*WIN32DLL*/
-static unsigned char hold_buf[HOLD_SIZE*2];
+static nkf_char hold_buf[HOLD_SIZE*2];
static int hold_count = 0;
static nkf_char
push_hold_buf(nkf_char c2)
{
if (hold_count >= HOLD_SIZE*2)
return (EOF);
- hold_buf[hold_count++] = (unsigned char)c2;
+ hold_buf[hold_count++] = c2;
return ((hold_count >= HOLD_SIZE*2) ? EOF : hold_count);
}
hold_index = 0;
while (hold_index < hold_count){
c1 = hold_buf[hold_index++];
- if (c1 <= DEL){
+ if (nkf_char_unicode_p(c1)) {
+ (*oconv)(0, c1);
+ continue;
+ }
+ else if (c1 <= DEL){
(*iconv)(0, c1, 0);
continue;
}else if (iconv == s_iconv && 0xa1 <= c1 && c1 <= 0xdf){
set_iconv(TRUE, w_iconv32);
}
if (iconv == w_iconv32) {
+ input_bom_f = TRUE;
input_endian = ENDIAN_BIG;
return;
}
set_iconv(TRUE, w_iconv);
}
if (iconv == w_iconv) {
+ input_bom_f = TRUE;
return;
}
(*i_ungetc)(0xBF,f);
}
if (iconv == w_iconv16) {
input_endian = ENDIAN_BIG;
+ input_bom_f = TRUE;
return;
}
(*i_ungetc)(0xFF,f);
}
if (iconv == w_iconv32) {
input_endian = ENDIAN_LITTLE;
+ input_bom_f = TRUE;
return;
}
(*i_ungetc)(0x00,f);
}
if (iconv == w_iconv16) {
input_endian = ENDIAN_LITTLE;
+ input_bom_f = TRUE;
return;
}
(*i_ungetc)(0xFE,f);
/* \e$B3:Ev$9$k%3!<%I$NM%@hEY$r>e$2$k$?$a$NL\0u\e(B */
nkf_char (*mime_priority_func[])(nkf_char c2, nkf_char c1, nkf_char c0) = {
- e_iconv, s_iconv, 0, 0, 0, 0,
+ e_iconv, s_iconv, 0, 0, 0, 0, 0,
#if defined(UTF8_INPUT_ENABLE)
w_iconv, w_iconv,
#endif
if (guess_f == 1) {
printf("%s\n", input_codename);
} else {
- printf("%s%s\n",
+ printf("%s%s%s%s\n",
input_codename,
+ iconv != w_iconv16 && iconv != w_iconv32 ? "" :
+ input_endian == ENDIAN_LITTLE ? " LE" :
+ input_endian == ENDIAN_BIG ? " BE" :
+ "[BUG]",
+ input_bom_f ? " (BOM)" : "",
input_eol == CR ? " (CR)" :
input_eol == LF ? " (LF)" :
input_eol == CRLF ? " (CRLF)" :
nkf_char (*u)(nkf_char c ,FILE *f) = i_nungetc;
int i = 0, j;
nkf_char buf[12];
- long c = -1;
+ nkf_char c = -1;
buf[i] = (*g)(f);
if (buf[i] == '&'){
#define MIMEOUT_BUF_LENGTH 74
static struct {
- char buf[MIMEOUT_BUF_LENGTH+1];
+ unsigned char buf[MIMEOUT_BUF_LENGTH+1];
int count;
} mimeout_state;
base64_count = 1;
}
} else {
- if (!(c2 == 0 && (c1 == CR || c1 == LF)) &&
- base64_count + mimeout_state.count/3*4> 66) {
+ if ((c2 != 0 || c1 > DEL) && base64_count + mimeout_state.count/3*4> 66) {
(*o_base64conv)(EOF,0);
oconv_newline(o_base64conv);
(*o_base64conv)(0,SP);
i = 0;
for (; i < mimeout_state.count - len; ++i) {
- if (!strncmp(mimeout_state.buf+i, str, len)) {
+ if (!strncmp((char *)(mimeout_state.buf+i), str, len)) {
i += len - 2;
break;
}
broken_f = FALSE;
iso8859_f = FALSE;
mimeout_f = FALSE;
- x0201_f = X0201_DEFAULT;
+ x0201_f = NKF_UNSPECIFIED;
iso2022jp_f = FALSE;
#if defined(UTF8_INPUT_ENABLE) || defined(UTF8_OUTPUT_ENABLE)
ms_ucs_map_f = UCS_MAP_ASCII;
if (nkf_enc_unicode_p(output_encoding))
output_mode = UTF_8;
+ if (x0201_f == NKF_UNSPECIFIED) {
+ x0201_f = X0201_DEFAULT;
+ }
+
/* replace continucation module, from output side */
/* output redicrection */
#define NEXT continue /* no output, get next */
#define SKIP c2=0;continue /* no output, get next */
#define MORE c2=c1;continue /* need one more byte */
-#define SEND ; /* output c1 and c2, get next */
+#define SEND (void)0 /* output c1 and c2, get next */
#define LAST break /* end of loop, go closing */
#define set_input_mode(mode) do { \
input_mode = mode; \
else if (iconv == w_iconv16) {
while ((c1 = (*i_getc)(f)) != EOF &&
(c2 = (*i_getc)(f)) != EOF) {
- if (nkf_iconv_utf_16(c1, c2, 0, 0) == -2 &&
+ if (nkf_iconv_utf_16(c1, c2, 0, 0) == NKF_ICONV_NEED_TWO_MORE_BYTES &&
(c3 = (*i_getc)(f)) != EOF &&
(c4 = (*i_getc)(f)) != EOF) {
nkf_iconv_utf_16(c1, c2, c3, c4);
/*
* int options(unsigned char *cp)
- *
+ *
* return values:
* 0: success
* -1: ArgumentError
option_mode = 1;
return 0;
}
- for (i=0;i<sizeof(long_option)/sizeof(long_option[0]);i++) {
+ for (i=0;i<(int)(sizeof(long_option)/sizeof(long_option[0]));i++) {
p = (unsigned char *)long_option[i].name;
for (j=0;*p && *p != '=' && *p == cp[j];p++, j++);
if (*p == cp[j] || cp[j] == SP){
break;
#endif
#ifdef UTF8_OUTPUT_ENABLE
- case 'w': /* UTF-8 output */
+ case 'w': /* UTF-{8,16,32} output */
if (cp[0] == '8') {
cp++;
if (cp[0] == '0'){
if (cp[0]=='L') {
cp++;
output_endian = ENDIAN_LITTLE;
+ output_bom_f = TRUE;
} else if (cp[0] == 'B') {
cp++;
+ output_bom_f = TRUE;
}
if (cp[0] == '0'){
+ output_bom_f = FALSE;
cp++;
enc_idx = enc_idx == UTF_16
? (output_endian == ENDIAN_LITTLE ? UTF_16LE : UTF_16BE)
: (output_endian == ENDIAN_LITTLE ? UTF_32LE : UTF_32BE);
} else {
- output_bom_f = TRUE;
enc_idx = enc_idx == UTF_16
? (output_endian == ENDIAN_LITTLE ? UTF_16LE_BOM : UTF_16BE_BOM)
: (output_endian == ENDIAN_LITTLE ? UTF_32LE_BOM : UTF_32BE_BOM);