/*
* Copyright (c) 1987, Fujitsu LTD. (Itaru ICHIKAWA).
- * Copyright (c) 1996-2009, The nkf Project.
+ * Copyright (c) 1996-2010, The nkf Project.
*
* This software is provided 'as-is', without any express or implied
* warranty. In no event will the authors be held liable for any damages
*
* 3. This notice may not be removed or altered from any source distribution.
*/
-#define NKF_VERSION "2.0.9"
-#define NKF_RELEASE_DATE "2009-01-20"
+#define NKF_VERSION "2.1.3"
+#define NKF_RELEASE_DATE "2012-03-17"
#define COPY_RIGHT \
"Copyright (C) 1987, FUJITSU LTD. (I.Ichikawa).\n" \
- "Copyright (C) 1996-2009, The nkf Project."
+ "Copyright (C) 1996-2012, The nkf Project."
#include "config.h"
#include "nkf.h"
} encoding_name_to_id_table[] = {
{"US-ASCII", ASCII},
{"ASCII", ASCII},
+ {"646", ASCII},
+ {"ROMAN8", ASCII},
{"ISO-2022-JP", ISO_2022_JP},
{"ISO2022JP-CP932", CP50220},
{"CP50220", CP50220},
{"ISO-2022-JP-2004", ISO_2022_JP_2004},
{"SHIFT_JIS", SHIFT_JIS},
{"SJIS", SHIFT_JIS},
+ {"MS_Kanji", SHIFT_JIS},
+ {"PCK", SHIFT_JIS},
{"WINDOWS-31J", WINDOWS_31J},
{"CSWINDOWS31J", WINDOWS_31J},
{"CP932", WINDOWS_31J},
&& (c != '(') && (c != ')') && (c != '.') && (c != 0x22)))
#define is_ibmext_in_sjis(c2) (CP932_TABLE_BEGIN <= c2 && c2 <= CP932_TABLE_END)
-#define nkf_byte_jisx0201_katakana_p(c) (SP <= c && c < (0xE0&0x7F))
+#define nkf_byte_jisx0201_katakana_p(c) (SP <= c && c <= 0x5F)
#define HOLD_SIZE 1024
#if defined(INT_IS_SHORT)
/* ignore ZERO WIDTH NO-BREAK SPACE */
static int no_best_fit_chars_f = FALSE;
static int input_endian = ENDIAN_BIG;
+static int input_bom_f = FALSE;
static nkf_char unicode_subchar = '?'; /* the regular substitution character */
static void (*encode_fallback)(nkf_char c) = NULL;
static void w_status(struct input_code *, nkf_char);
static unsigned char stdobuf[IOBUF_SIZE];
#endif
+#define NKF_UNSPECIFIED (-TRUE)
+
/* flags */
static int unbuf_f = FALSE;
static int estab_f = FALSE;
static int broken_f = FALSE; /* convert ESC-less broken JIS */
static int iso8859_f = FALSE; /* ISO8859 through */
static int mimeout_f = FALSE; /* base64 mode */
-static int x0201_f = X0201_DEFAULT; /* convert JIS X 0201 */
+static int x0201_f = NKF_UNSPECIFIED; /* convert JIS X 0201 */
static int iso2022jp_f = FALSE; /* replace non ISO-2022-JP with GETA */
#ifdef UNICODE_NORMALIZATION
{"Shift_JIS", 0, 0, 0, {0, 0, 0}, s_status, s_iconv, 0},
#ifdef UTF8_INPUT_ENABLE
{"UTF-8", 0, 0, 0, {0, 0, 0}, w_status, w_iconv, 0},
+ {"UTF-16", 0, 0, 0, {0, 0, 0}, NULL, w_iconv16, 0},
+ {"UTF-32", 0, 0, 0, {0, 0, 0}, NULL, w_iconv32, 0},
#endif
- {0}
+ {NULL, 0, 0, 0, {0, 0, 0}, NULL, NULL, 0}
};
static int mimeout_mode = 0; /* 0, -1, 'Q', 'B', 1, 2 */
typedef struct {
long capa;
long len;
- unsigned char *ptr;
+ nkf_char *ptr;
} nkf_buf_t;
static nkf_buf_t *
nkf_buf_new(int length)
{
nkf_buf_t *buf = nkf_xmalloc(sizeof(nkf_buf_t));
- buf->ptr = nkf_xmalloc(length);
+ buf->ptr = nkf_xmalloc(sizeof(nkf_char) * length);
buf->capa = length;
buf->len = 0;
return buf;
-}
+}
#if 0
static void
#define nkf_buf_length(buf) ((buf)->len)
#define nkf_buf_empty_p(buf) ((buf)->len == 0)
-static unsigned char
+static nkf_char
nkf_buf_at(nkf_buf_t *buf, int index)
{
assert(index <= buf->len);
}
static void
-nkf_buf_push(nkf_buf_t *buf, unsigned char c)
+nkf_buf_push(nkf_buf_t *buf, nkf_char c)
{
if (buf->capa <= buf->len) {
exit(EXIT_FAILURE);
buf->ptr[buf->len++] = c;
}
-static unsigned char
+static nkf_char
nkf_buf_pop(nkf_buf_t *buf)
{
assert(!nkf_buf_empty_p(buf));
{
fprintf(HELP_OUTPUT,
"Usage: nkf -[flags] [--] [in file] .. [out file for -O flag]\n"
- " j,s,e,w Output code is ISO-2022-JP, Shift JIS, EUC-JP, UTF-8N\n"
#ifdef UTF8_OUTPUT_ENABLE
- " After 'w' you can add more options. -w[ 8 [0], 16 [[BL] [0]] ]\n"
+ " j/s/e/w Specify output encoding ISO-2022-JP, Shift_JIS, EUC-JP\n"
+ " UTF options is -w[8[0],{16,32}[{B,L}[0]]]\n"
+#else
#endif
- " J,S,E,W Input assumption is JIS 7 bit , Shift JIS, EUC-JP, UTF-8\n"
#ifdef UTF8_INPUT_ENABLE
- " After 'W' you can add more options. -W[ 8, 16 [BL] ] \n"
+ " J/S/E/W Specify input encoding ISO-2022-JP, Shift_JIS, EUC-JP\n"
+ " UTF option is -W[8,[16,32][B,L]]\n"
+#else
+ " J/S/E Specify output encoding ISO-2022-JP, Shift_JIS, EUC-JP\n"
#endif
);
fprintf(HELP_OUTPUT,
- " m[BQSN0] MIME decode [B:base64,Q:quoted,S:strict,N:non-strict,0:no decode]\n"
+ " m[BQSN0] MIME decode [B:base64,Q:quoted,S:strict,N:nonstrict,0:no decode]\n"
" M[BQ] MIME encode [B:base64 Q:quoted]\n"
" f/F Folding: -f60 or -f or -f60-10 (fold margin 10) F preserve nl\n"
);
" Z[0-4] Default/0: Convert JISX0208 Alphabet to ASCII\n"
" 1: Kankaku to one space 2: to two spaces 3: HTML Entity\n"
" 4: JISX0208 Katakana to JISX0201 Katakana\n"
- " X,x Assume X0201 kana in MS-Kanji, -x preserves X0201\n"
+ " X,x Convert Halfwidth Katakana to Fullwidth or preserve it\n"
);
fprintf(HELP_OUTPUT,
" O Output to File (DEFAULT 'nkf.out')\n"
" L[uwm] Line mode u:LF w:CRLF m:CR (DEFAULT noconversion)\n"
);
fprintf(HELP_OUTPUT,
- "Long name options\n"
- " --ic=<input codeset> --oc=<output codeset>\n"
- " Specify the input or output codeset\n"
- " --hiragana --katakana --katakana-hiragana\n"
- " To Hiragana/Katakana Conversion\n"
+ " --ic=<encoding> Specify the input encoding\n"
+ " --oc=<encoding> Specify the output encoding\n"
+ " --hiragana --katakana Hiragana/Katakana Conversion\n"
+ " --katakana-hiragana Converts each other\n"
);
fprintf(HELP_OUTPUT,
#ifdef INPUT_OPTION
- " --cap-input, --url-input Convert hex after ':' or '%%'\n"
+ " --{cap, url}-input Convert hex after ':' or '%%'\n"
#endif
#ifdef NUMCHAR_OPTION
- " --numchar-input Convert Unicode Character Reference\n"
+ " --numchar-input Convert Unicode Character Reference\n"
#endif
#ifdef UTF8_INPUT_ENABLE
" --fb-{skip, html, xml, perl, java, subchar}\n"
- " Specify how nkf handles unassigned characters\n"
+ " Specify unassigned character's replacement\n"
#endif
);
fprintf(HELP_OUTPUT,
#ifdef OVERWRITE
- " --in-place[=SUF] Overwrite original listed files by filtered result\n"
- " --overwrite[=SUF] in-place and preserve timestamp of original files\n"
+ " --in-place[=SUF] Overwrite original files\n"
+ " --overwrite[=SUF] Preserve timestamp of original files\n"
#endif
- " -g --guess Guess the input code\n"
- " -v --version print the version\n"
- " --help/-V print this help / configuration\n"
+ " -g --guess Guess the input code\n"
+ " -v --version Print the version\n"
+ " --help/-V Print this help / configuration\n"
);
version();
}
int shift = 20;
c &= VALUE_MASK;
while(shift >= 0){
- if(c >= 1<<shift){
+ if(c >= NKF_INT32_C(1)<<shift){
while(shift >= 0){
(*f)(0, bin2hex(c>>shift));
shift -= 4;
case ISO_8859_1:
iso8859_f = TRUE;
break;
- case CP50220:
case CP50221:
case CP50222:
+ if (x0201_f == NKF_UNSPECIFIED) x0201_f = FALSE; /* -x specified implicitly */
+ case CP50220:
#ifdef SHIFTJIS_CP932
cp51932_f = TRUE;
#endif
case SHIFT_JIS:
break;
case WINDOWS_31J:
+ if (x0201_f == NKF_UNSPECIFIED) x0201_f = FALSE; /* -x specified implicitly */
#ifdef SHIFTJIS_CP932
cp51932_f = TRUE;
#endif
case EUCJP_NKF:
break;
case CP51932:
+ if (x0201_f == NKF_UNSPECIFIED) x0201_f = FALSE; /* -x specified implicitly */
#ifdef SHIFTJIS_CP932
cp51932_f = TRUE;
#endif
#endif
break;
case EUCJP_MS:
+ if (x0201_f == NKF_UNSPECIFIED) x0201_f = FALSE; /* -x specified implicitly */
#ifdef SHIFTJIS_CP932
cp51932_f = FALSE;
#endif
#endif
break;
case EUCJP_ASCII:
+ if (x0201_f == NKF_UNSPECIFIED) x0201_f = FALSE; /* -x specified implicitly */
#ifdef SHIFTJIS_CP932
cp51932_f = FALSE;
#endif
{
switch (nkf_enc_to_index(enc)) {
case CP50220:
- x0201_f = TRUE;
#ifdef SHIFTJIS_CP932
if (cp932inv_f == TRUE) cp932inv_f = FALSE;
#endif
#endif
break;
case CP50221:
+ if (x0201_f == NKF_UNSPECIFIED) x0201_f = FALSE; /* -x specified implicitly */
#ifdef SHIFTJIS_CP932
if (cp932inv_f == TRUE) cp932inv_f = FALSE;
#endif
ms_ucs_map_f = UCS_MAP_CP932;
#endif
break;
+ case ISO_2022_JP:
+#ifdef SHIFTJIS_CP932
+ if (cp932inv_f == TRUE) cp932inv_f = FALSE;
+#endif
+ break;
case ISO_2022_JP_1:
x0212_f = TRUE;
#ifdef SHIFTJIS_CP932
case SHIFT_JIS:
break;
case WINDOWS_31J:
+ if (x0201_f == NKF_UNSPECIFIED) x0201_f = FALSE; /* -x specified implicitly */
#ifdef UTF8_OUTPUT_ENABLE
ms_ucs_map_f = UCS_MAP_CP932;
#endif
#endif
break;
case CP51932:
+ if (x0201_f == NKF_UNSPECIFIED) x0201_f = FALSE; /* -x specified implicitly */
#ifdef SHIFTJIS_CP932
if (cp932inv_f == TRUE) cp932inv_f = FALSE;
#endif
#endif
break;
case EUCJP_MS:
+ if (x0201_f == NKF_UNSPECIFIED) x0201_f = FALSE; /* -x specified implicitly */
x0212_f = TRUE;
#ifdef UTF8_OUTPUT_ENABLE
ms_ucs_map_f = UCS_MAP_MS;
#endif
break;
case EUCJP_ASCII:
+ if (x0201_f == NKF_UNSPECIFIED) x0201_f = FALSE; /* -x specified implicitly */
x0212_f = TRUE;
#ifdef UTF8_OUTPUT_ENABLE
ms_ucs_map_f = UCS_MAP_ASCII;
output_endian = ENDIAN_LITTLE;
output_bom_f = TRUE;
break;
+ case UTF_32:
case UTF_32BE_BOM:
output_bom_f = TRUE;
break;
*p3 = 0x80 | ( val & 0x3f);
*p4 = 0;
} else if (nkf_char_unicode_value_p(val)) {
- *p1 = 0xe0 | (val >> 16);
+ *p1 = 0xf0 | (val >> 18);
*p2 = 0x80 | ((val >> 12) & 0x3f);
*p3 = 0x80 | ((val >> 6) & 0x3f);
*p4 = 0x80 | ( val & 0x3f);
return 0;
}
-#define NKF_ICONV_NEED_ONE_MORE_BYTE -1
-#define NKF_ICONV_NEED_TWO_MORE_BYTES -2
+#define NKF_ICONV_NEED_ONE_MORE_BYTE (size_t)-1
+#define NKF_ICONV_NEED_TWO_MORE_BYTES (size_t)-2
#define UTF16_TO_UTF32(lead, trail) (((lead) << 10) + (trail) - NKF_INT32_C(0x35FDC00))
static size_t
nkf_iconv_utf_16(nkf_char c1, nkf_char c2, nkf_char c3, nkf_char c4)
static nkf_char
w_iconv16(nkf_char c2, nkf_char c1, nkf_char c0)
{
- return 0;
+ (*oconv)(c2, c1);
+ return 16; /* different from w_iconv32 */
}
static nkf_char
w_iconv32(nkf_char c2, nkf_char c1, nkf_char c0)
{
- return 0;
+ (*oconv)(c2, c1);
+ return 32; /* different from w_iconv16 */
}
static size_t
}
#endif /*WIN32DLL*/
-static unsigned char hold_buf[HOLD_SIZE*2];
+static nkf_char hold_buf[HOLD_SIZE*2];
static int hold_count = 0;
static nkf_char
push_hold_buf(nkf_char c2)
{
if (hold_count >= HOLD_SIZE*2)
return (EOF);
- hold_buf[hold_count++] = (unsigned char)c2;
+ hold_buf[hold_count++] = c2;
return ((hold_count >= HOLD_SIZE*2) ? EOF : hold_count);
}
static int
-h_conv(FILE *f, int c1, int c2)
+h_conv(FILE *f, nkf_char c1, nkf_char c2)
{
- int ret, c4, c3;
+ int ret;
int hold_index;
-
+ nkf_char c3, c4;
/** it must NOT be in the kanji shifte sequence */
/** it must NOT be written in JIS7 */
hold_index = 0;
while (hold_index < hold_count){
c1 = hold_buf[hold_index++];
- if (c1 <= DEL){
+ if (nkf_char_unicode_p(c1)) {
+ (*oconv)(0, c1);
+ continue;
+ }
+ else if (c1 <= DEL){
(*iconv)(0, c1, 0);
continue;
}else if (iconv == s_iconv && 0xa1 <= c1 && c1 <= 0xdf){
} else if ((c3 = (*i_getc)(f)) == EOF) {
ret = EOF;
break;
- } else {
- code_status(c3);
- if (hold_index < hold_count){
- c4 = hold_buf[hold_index++];
- } else if ((c4 = (*i_getc)(f)) == EOF) {
- c3 = ret = EOF;
- break;
- } else {
- code_status(c4);
- (*iconv)(c1, c2, (c3<<8)|c4);
- }
}
+ code_status(c3);
+ if (hold_index < hold_count){
+ c4 = hold_buf[hold_index++];
+ } else if ((c4 = (*i_getc)(f)) == EOF) {
+ c3 = ret = EOF;
+ break;
+ }
+ code_status(c4);
+ (*iconv)(c1, c2, (c3<<8)|c4);
break;
case -1:
/* 3 bytes EUC or UTF-8 */
set_iconv(TRUE, w_iconv32);
}
if (iconv == w_iconv32) {
+ input_bom_f = TRUE;
input_endian = ENDIAN_BIG;
return;
}
set_iconv(TRUE, w_iconv);
}
if (iconv == w_iconv) {
+ input_bom_f = TRUE;
return;
}
(*i_ungetc)(0xBF,f);
}
if (iconv == w_iconv16) {
input_endian = ENDIAN_BIG;
+ input_bom_f = TRUE;
return;
}
(*i_ungetc)(0xFF,f);
}
if (iconv == w_iconv32) {
input_endian = ENDIAN_LITTLE;
+ input_bom_f = TRUE;
return;
}
(*i_ungetc)(0x00,f);
}
if (iconv == w_iconv16) {
input_endian = ENDIAN_LITTLE;
+ input_bom_f = TRUE;
return;
}
(*i_ungetc)(0xFE,f);
else if (c2 != 0 || c1 != LF) (*o_eol_conv)(c2, c1);
}
+static void
+put_newline(void (*func)(nkf_char))
+{
+ switch (eolmode_f ? eolmode_f : DEFAULT_NEWLINE) {
+ case CRLF:
+ (*func)(0x0D);
+ (*func)(0x0A);
+ break;
+ case CR:
+ (*func)(0x0D);
+ break;
+ case LF:
+ (*func)(0x0A);
+ break;
+ }
+}
+
+static void
+oconv_newline(void (*func)(nkf_char, nkf_char))
+{
+ switch (eolmode_f ? eolmode_f : DEFAULT_NEWLINE) {
+ case CRLF:
+ (*func)(0, 0x0D);
+ (*func)(0, 0x0A);
+ break;
+ case CR:
+ (*func)(0, 0x0D);
+ break;
+ case LF:
+ (*func)(0, 0x0A);
+ break;
+ }
+}
+
/*
Return value of fold_conv()
/* terminator process */
switch(fold_state) {
case LF:
- OCONV_NEWLINE((*o_fconv));
+ oconv_newline(o_fconv);
(*o_fconv)(c2,c1);
break;
case 0:
return;
case CR:
- OCONV_NEWLINE((*o_fconv));
+ oconv_newline(o_fconv);
break;
case TAB:
case SP:
(const unsigned char *)"\075?ISO-8859-1?Q?",
(const unsigned char *)"\075?ISO-8859-1?B?",
(const unsigned char *)"\075?ISO-2022-JP?B?",
+ (const unsigned char *)"\075?ISO-2022-JP?B?",
(const unsigned char *)"\075?ISO-2022-JP?Q?",
#if defined(UTF8_INPUT_ENABLE)
(const unsigned char *)"\075?UTF-8?B?",
/* \e$B3:Ev$9$k%3!<%I$NM%@hEY$r>e$2$k$?$a$NL\0u\e(B */
nkf_char (*mime_priority_func[])(nkf_char c2, nkf_char c1, nkf_char c0) = {
- e_iconv, s_iconv, 0, 0, 0, 0,
+ e_iconv, s_iconv, 0, 0, 0, 0, 0,
#if defined(UTF8_INPUT_ENABLE)
w_iconv, w_iconv,
#endif
};
static const nkf_char mime_encode[] = {
- EUC_JP, SHIFT_JIS, ISO_8859_1, ISO_8859_1, JIS_X_0208, JIS_X_0201_1976_K,
+ EUC_JP, SHIFT_JIS, ISO_8859_1, ISO_8859_1, JIS_X_0208, JIS_X_0201_1976_K, JIS_X_0201_1976_K,
#if defined(UTF8_INPUT_ENABLE)
UTF_8, UTF_8,
#endif
};
static const nkf_char mime_encode_method[] = {
- 'B', 'B','Q', 'B', 'B', 'Q',
+ 'B', 'B','Q', 'B', 'B', 'B', 'Q',
#if defined(UTF8_INPUT_ENABLE)
'B', 'Q',
#endif
if (guess_f == 1) {
printf("%s\n", input_codename);
} else {
- printf("%s%s\n",
+ printf("%s%s%s%s\n",
input_codename,
+ iconv != w_iconv16 && iconv != w_iconv32 ? "" :
+ input_endian == ENDIAN_LITTLE ? " LE" :
+ input_endian == ENDIAN_BIG ? " BE" :
+ "[BUG]",
+ input_bom_f ? " (BOM)" : "",
input_eol == CR ? " (CR)" :
input_eol == LF ? " (LF)" :
input_eol == CRLF ? " (CRLF)" :
nkf_char (*u)(nkf_char c ,FILE *f) = i_nungetc;
int i = 0, j;
nkf_char buf[12];
- long c = -1;
+ nkf_char c = -1;
buf[i] = (*g)(f);
if (buf[i] == '&'){
if (c == EOF || c > 0xFF || (c & 0xc0) == 0x80) return c;
- nkf_buf_push(buf, (unsigned char)c);
+ nkf_buf_push(buf, c);
do {
while (lower <= upper) {
int mid = (lower+upper) / 2;
}
if (c1=='='&&c2<SP) { /* this is soft wrap */
while((c1 = (*i_mgetc)(f)) <=SP) {
- if ((c1 = (*i_mgetc)(f)) == EOF) return (EOF);
+ if (c1 == EOF) return (EOF);
}
mime_decode_mode = 'Q'; /* still in MIME */
goto restart_mime_q;
#define MIMEOUT_BUF_LENGTH 74
static struct {
- char buf[MIMEOUT_BUF_LENGTH+1];
+ unsigned char buf[MIMEOUT_BUF_LENGTH+1];
int count;
} mimeout_state;
(*o_mputc)(mimeout_state.buf[i]);
i++;
}
- PUT_NEWLINE((*o_mputc));
+ put_newline(o_mputc);
(*o_mputc)(SP);
base64_count = 1;
if (mimeout_state.count>0 && nkf_isspace(mimeout_state.buf[i])) {
if (c2 == EOF){
if (base64_count + mimeout_state.count/3*4> 73){
(*o_base64conv)(EOF,0);
- OCONV_NEWLINE((*o_base64conv));
+ oconv_newline(o_base64conv);
(*o_base64conv)(0,SP);
base64_count = 1;
}
} else {
- if (base64_count + mimeout_state.count/3*4> 66) {
+ if ((c2 != 0 || c1 > DEL) && base64_count + mimeout_state.count/3*4> 66) {
(*o_base64conv)(EOF,0);
- OCONV_NEWLINE((*o_base64conv));
+ oconv_newline(o_base64conv);
(*o_base64conv)(0,SP);
base64_count = 1;
mimeout_mode = -1;
mimeout_mode = (output_mode==ASCII ||output_mode == ISO_8859_1) ? 'Q' : 'B';
open_mime(output_mode);
(*o_base64conv)(EOF,0);
- OCONV_NEWLINE((*o_base64conv));
+ oconv_newline(o_base64conv);
(*o_base64conv)(0,SP);
base64_count = 1;
mimeout_mode = -1;
if (base64_count > 71){
if (c!=CR && c!=LF) {
(*o_mputc)('=');
- PUT_NEWLINE((*o_mputc));
+ put_newline(o_mputc);
}
base64_count = 0;
}
}else{
if (base64_count > 71){
eof_mime();
- PUT_NEWLINE((*o_mputc));
+ put_newline(o_mputc);
base64_count = 0;
}
if (c == EOF) { /* c==EOF */
} else if (c <= SP) {
close_mime();
if (base64_count > 70) {
- PUT_NEWLINE((*o_mputc));
+ put_newline(o_mputc);
base64_count = 0;
}
if (!nkf_isblank(c)) {
} else {
if (base64_count > 70) {
close_mime();
- PUT_NEWLINE((*o_mputc));
+ put_newline(o_mputc);
(*o_mputc)(SP);
base64_count = 1;
open_mime(output_mode);
return;
}
}
- (*o_mputc)(c);
- base64_count++;
+ if (c != 0x1B) {
+ (*o_mputc)(c);
+ base64_count++;
+ return;
+ }
}
- return;
}
if (mimeout_mode <= 0) {
- if (c <= DEL && (output_mode==ASCII ||output_mode == ISO_8859_1)) {
+ if (c <= DEL && (output_mode==ASCII || output_mode == ISO_8859_1 ||
+ output_mode == UTF_8)) {
if (nkf_isspace(c)) {
int flag = 0;
if (mimeout_mode == -1) {
i = 0;
for (; i < mimeout_state.count - len; ++i) {
- if (!strncmp(mimeout_state.buf+i, str, len)) {
+ if (!strncmp((char *)(mimeout_state.buf+i), str, len)) {
i += len - 2;
break;
}
}
if (i == 0 || i == mimeout_state.count - len) {
- PUT_NEWLINE((*o_mputc));
+ put_newline(o_mputc);
base64_count = 0;
if (!nkf_isspace(mimeout_state.buf[0])){
(*o_mputc)(SP);
for (j = 0; j <= i; ++j) {
(*o_mputc)(mimeout_state.buf[j]);
}
- PUT_NEWLINE((*o_mputc));
+ put_newline(o_mputc);
base64_count = 1;
for (; j <= mimeout_state.count; ++j) {
mimeout_state.buf[j - i] = mimeout_state.buf[j];
}
}else{
/* mimeout_mode == 'B', 1, 2 */
- if ( c<=DEL && (output_mode==ASCII ||output_mode == ISO_8859_1)) {
+ if (c <= DEL && (output_mode==ASCII || output_mode == ISO_8859_1 ||
+ output_mode == UTF_8)) {
if (lastchar == CR || lastchar == LF){
if (nkf_isblank(c)) {
for (i=0;i<mimeout_state.count;i++) {
mimeout_addchar(mimeout_state.buf[i]);
}
mimeout_state.count = 0;
- } else if (SP<c && c<DEL) {
+ } else {
eof_mime();
for (i=0;i<mimeout_state.count;i++) {
(*o_mputc)(mimeout_state.buf[i]);
broken_f = FALSE;
iso8859_f = FALSE;
mimeout_f = FALSE;
- x0201_f = X0201_DEFAULT;
+ x0201_f = NKF_UNSPECIFIED;
iso2022jp_f = FALSE;
#if defined(UTF8_INPUT_ENABLE) || defined(UTF8_OUTPUT_ENABLE)
ms_ucs_map_f = UCS_MAP_ASCII;
set_output_encoding(output_encoding);
oconv = nkf_enc_to_oconv(output_encoding);
o_putc = std_putc;
+ if (nkf_enc_unicode_p(output_encoding))
+ output_mode = UTF_8;
+
+ if (x0201_f == NKF_UNSPECIFIED) {
+ x0201_f = X0201_DEFAULT;
+ }
/* replace continucation module, from output side */
#define NEXT continue /* no output, get next */
#define SKIP c2=0;continue /* no output, get next */
#define MORE c2=c1;continue /* need one more byte */
-#define SEND ; /* output c1 and c2, get next */
+#define SEND (void)0 /* output c1 and c2, get next */
#define LAST break /* end of loop, go closing */
#define set_input_mode(mode) do { \
input_mode = mode; \
(c4 = (*i_getc)(f)) != EOF) {
nkf_iconv_utf_32(c1, c2, c3, c4);
}
- (*i_ungetc)(EOF, f);
+ goto finished;
}
else if (iconv == w_iconv16) {
while ((c1 = (*i_getc)(f)) != EOF &&
(c2 = (*i_getc)(f)) != EOF) {
- if (nkf_iconv_utf_16(c1, c2, 0, 0) == -2 &&
+ if (nkf_iconv_utf_16(c1, c2, 0, 0) == NKF_ICONV_NEED_TWO_MORE_BYTES &&
(c3 = (*i_getc)(f)) != EOF &&
(c4 = (*i_getc)(f)) != EOF) {
nkf_iconv_utf_16(c1, c2, c3, c4);
}
}
- (*i_ungetc)(EOF, f);
+ goto finished;
}
#endif
if (input_mode == JIS_X_0208 && DEL <= c1 && c1 < 0x92) {
/* CP5022x */
MORE;
+ }else if (input_codename && input_codename[0] == 'I' &&
+ 0xA1 <= c1 && c1 <= 0xDF) {
+ /* JIS X 0201 Katakana in 8bit JIS */
+ c2 = JIS_X_0201_1976_K;
+ c1 &= 0x7f;
+ SEND;
} else if (c1 > DEL) {
/* 8 bit code */
if (!estab_f && !iso8859_f) {
SKIP;
} else if (c1 == ESC && (!is_8bit || mime_decode_mode)) {
if ((c1 = (*i_getc)(f)) == EOF) {
- /* (*oconv)(0, ESC); don't send bogus code */
+ (*oconv)(0, ESC);
LAST;
}
else if (c1 == '&') {
} else if (c1 == ESC && iconv == s_iconv) {
/* ESC in Shift_JIS */
if ((c1 = (*i_getc)(f)) == EOF) {
- /* (*oconv)(0, ESC); don't send bogus code */
+ (*oconv)(0, ESC);
LAST;
} else if (c1 == '$') {
/* J-PHONE emoji */
/* goto next_word */
}
+finished:
/* epilogue */
(*iconv)(EOF, 0, 0);
if (!input_codename)
/*
* int options(unsigned char *cp)
- *
+ *
* return values:
* 0: success
* -1: ArgumentError
option_mode = 1;
return 0;
}
- for (i=0;i<sizeof(long_option)/sizeof(long_option[0]);i++) {
+ for (i=0;i<(int)(sizeof(long_option)/sizeof(long_option[0]));i++) {
p = (unsigned char *)long_option[i].name;
for (j=0;*p && *p != '=' && *p == cp[j];p++, j++);
if (*p == cp[j] || cp[j] == SP){
cp_back = cp;
cp = (unsigned char *)long_option[i].alias;
}else{
+#ifndef PERL_XS
if (strcmp(long_option[i].name, "help") == 0){
usage();
exit(EXIT_SUCCESS);
}
+#endif
if (strcmp(long_option[i].name, "ic=") == 0){
enc = nkf_enc_find((char *)p);
if (!enc) continue;
break;
#endif
#ifdef UTF8_OUTPUT_ENABLE
- case 'w': /* UTF-8 output */
+ case 'w': /* UTF-{8,16,32} output */
if (cp[0] == '8') {
cp++;
if (cp[0] == '0'){
if (cp[0]=='L') {
cp++;
output_endian = ENDIAN_LITTLE;
+ output_bom_f = TRUE;
} else if (cp[0] == 'B') {
cp++;
- } else {
- output_encoding = nkf_enc_from_index(enc_idx);
- continue;
+ output_bom_f = TRUE;
}
if (cp[0] == '0'){
+ output_bom_f = FALSE;
cp++;
enc_idx = enc_idx == UTF_16
? (output_endian == ENDIAN_LITTLE ? UTF_16LE : UTF_16BE)
: (output_endian == ENDIAN_LITTLE ? UTF_32LE : UTF_32BE);
} else {
- output_bom_f = TRUE;
enc_idx = enc_idx == UTF_16
? (output_endian == ENDIAN_LITTLE ? UTF_16LE_BOM : UTF_16BE_BOM)
: (output_endian == ENDIAN_LITTLE ? UTF_32LE_BOM : UTF_32BE_BOM);
bit:3 Convert HTML Entity
bit:4 Convert JIS X 0208 Katakana to JIS X 0201 Katakana
*/
- while ('0'<= *cp && *cp <='9') {
+ while ('0'<= *cp && *cp <='4') {
alpha_f |= 1 << (*cp++ - '0');
}
- if (!alpha_f) alpha_f = 1;
+ alpha_f |= 1;
continue;
case 'x': /* Convert X0201 kana to X0208 or X0201 Conversion */
x0201_f = FALSE; /* No X0201->X0208 conversion */
OSDN Git Service
