diff --git a/include/sj4charset/sj4charset.h b/include/sj4charset/sj4charset.h index 9aaeb8a..72c0d79 100644 --- a/include/sj4charset/sj4charset.h +++ b/include/sj4charset/sj4charset.h @@ -23,8 +23,14 @@ int sj4_to_sjis(u_char* out, const u_char* in, int len); int sj4_to_utf8(u_char* out, const u_char* in, int len); int sj4_to_utf16(u_char* out, const u_char* in, int len); +int sj4_eucjp_read(const unsigned char* in, int* output); +int sj4_sjis_read(const unsigned char* in, int* output); +int sj4_utf8_read(const unsigned char* in, int* output); +int sj4_utf16_read(const unsigned char* in, int* output); + int sj4_charset_to(int type, void* output, const void* input, int len); int sj4_charset_from(int type, void* output, const void* input, int len); +int sj4_charset_read(int type, int* output, const void* input); #ifdef __cplusplus } diff --git a/include/sj4core/Struct.h b/include/sj4core/Struct.h index 12958df..1280499 100644 --- a/include/sj4core/Struct.h +++ b/include/sj4core/Struct.h @@ -80,6 +80,7 @@ typedef struct global { DICT* Jcurdict; DICTL* Jdictlist; u_char* Jdicbuf; + u_char* Jaskbase; u_char* Jidxbuf; TypeIdxOfs* Jidxofs; u_char* Jaskknj[MAXKNJASKNUMBER]; diff --git a/include/sj4core/sj_kanakan.h b/include/sj4core/sj_kanakan.h index 67df97d..b8c3dde 100644 --- a/include/sj4core/sj_kanakan.h +++ b/include/sj4core/sj_kanakan.h @@ -65,7 +65,27 @@ CLREC* alloc_clrec(SJ4_CONTEXT2); void free_clrec(SJ4_CONTEXT CLREC*); /* charsize.c */ +static __inline int codesize_inline(u_char code) { + switch(code & KANJIMODEMASK) { + case ZENHIRAASSYUKU: + case ZENKATAASSYUKU: + case KANJIASSYUKU: + case KANJISTREND: + return 1; + case LEADINGHANKAKU: + case OFFSETASSYUKU: + case AIATTRIBUTE: + case KANAKANCOST: + return 2; + default: + return 2; + } +} +#ifdef NO_CODESIZE_INLINE int codesize(u_char); +#else +#define codesize codesize_inline +#endif /* chrtbl.c */ extern u_char chrtbl[256]; @@ -139,7 +159,22 @@ int getkanji(SJ4_CONTEXT u_char*, int, u_char*, u_char*); void getrank(SJ4_CONTEXT2); /* hzstrlen.c */ +static __inline int euc_codesize_inline(u_char c) { + if((c & KANJIMODEMASK) == 0x90) { + return 1; + } else if(isknj1(c)) { + return 2; + } else if(c == SS3) { + return 3; + } else { + return 1; + } +} +#ifdef NO_EUC_CODESIZE_INLINE int euc_codesize(u_char c); +#else +#define euc_codesize euc_codesize_inline +#endif int hzstrlen(u_char*, int); /* init.c */ diff --git a/include/sj4core/sj_rename.h b/include/sj4core/sj_rename.h index 7694b5f..1226053 100644 --- a/include/sj4core/sj_rename.h +++ b/include/sj4core/sj_rename.h @@ -169,6 +169,7 @@ #define curdict (work_base->Jcurdict) #define dictlist (work_base->Jdictlist) #define dicbuf (work_base->Jdicbuf) +#define askbase (work_base->Jaskbase) #define idxbuf (work_base->Jidxbuf) #define idxofs (work_base->Jidxofs) #define askknj (work_base->Jaskknj) diff --git a/lib/sj4charset/conv.c b/lib/sj4charset/conv.c index 6c20b11..5032466 100644 --- a/lib/sj4charset/conv.c +++ b/lib/sj4charset/conv.c @@ -94,46 +94,60 @@ static int utf8_later(unsigned char c) { #define CAST_I32(x) ((int)(x)) +int sj4_utf8_read(const unsigned char* in, int* output) { + u_char c = in[0]; + u_int n = 0; + + if(c < 0x80) { + *output = c; + + return 1; + } else if(0xc2 <= c && c < 0xe0) { + if(!utf8_later(in[1])) return 0; + if((in[0] & 0x1e) == 0) return 0; + + n = CAST_I32(in[0] & 0x1f) << 6; + n |= CAST_I32(in[1] & 0x3f); + + *output = n; + + return 2; + } else if(0xe0 <= c && c < 0xf0) { + if(!utf8_later(in[1]) || !utf8_later(in[2])) return 0; + if((in[0] & 0x0f) == 0 && (in[1] & 0x20) == 0) return 0; + + n = CAST_I32(in[0] & 0x0f) << 12; + n |= CAST_I32(in[1] & 0x3f) << 6; + n |= CAST_I32(in[2] & 0x3f); + + *output = n; + + return 3; + } else if(0xf0 <= c && c < 0xf8) { + if(!utf8_later(in[1]) || !utf8_later(in[2]) || !utf8_later(in[3])) return 0; + if((in[0] & 0x07) == 0 && (in[1] & 0x30) == 0) return 0; + + n = CAST_I32(in[0] & 0x07) << 18; + n |= CAST_I32(in[1] & 0x3f) << 12; + n |= CAST_I32(in[2] & 0x3f) << 6; + n |= CAST_I32(in[3] & 0x3f); + + *output = n; + + return 4; + } + + return 0; +} + int sj4_from_utf8(u_char* out, const u_char* in, int len) { const u_char* o_in = in; u_char* o_out = out; while((in - o_in) < len) { - u_char c = *in; - u_int n = 0; + int n; - if(c < 0x80) { - n = c; - - in += 1; - } else if(0xc2 <= c && c < 0xe0) { - if(!utf8_later(in[1])) return 0; - if((in[0] & 0x1e) == 0) return 0; - - n = CAST_I32(in[0] & 0x1f) << 6; - n |= CAST_I32(in[1] & 0x3f); - - in += 2; - } else if(0xe0 <= c && c < 0xf0) { - if(!utf8_later(in[1]) || !utf8_later(in[2])) return 0; - if((in[0] & 0x0f) == 0 && (in[1] & 0x20) == 0) return 0; - - n = CAST_I32(in[0] & 0x0f) << 12; - n |= CAST_I32(in[1] & 0x3f) << 6; - n |= CAST_I32(in[2] & 0x3f); - - in += 3; - } else if(0xf0 <= c && c < 0xf8) { - if(!utf8_later(in[1]) || !utf8_later(in[2]) || !utf8_later(in[3])) return 0; - if((in[0] & 0x07) == 0 && (in[1] & 0x30) == 0) return 0; - - n = CAST_I32(in[0] & 0x07) << 18; - n |= CAST_I32(in[1] & 0x3f) << 12; - n |= CAST_I32(in[2] & 0x3f) << 6; - n |= CAST_I32(in[3] & 0x3f); - - in += 4; - } + in += sj4_utf8_read(in, &n); n = ucs_to_euc(n); @@ -151,39 +165,45 @@ static int is_low_urr(wchar_t ch) { return 0xDC00 <= ch && ch < 0xE000; } +int sj4_utf16_read(const unsigned char* in, int* output) { + const wchar_t* w_in = (const wchar_t*)in; + + if(is_high_surr(w_in[0])) { + if(is_low_urr(w_in[1])) { + *output = 0x10000 + CAST_I32(w_in[0] - 0xd800) * 0x400 + CAST_I32(w_in[1] - 0xdc00); + + return 2; + } else if(w_in[1] == 0) { + *output = w_in[0]; + + return 2; + } else { + return 0; + } + } else if(is_low_urr(w_in[0])) { + if(w_in[1] == 0) { + *output = w_in[0]; + + return 2; + } else { + return 0; + } + } else { + *output = w_in[0]; + + return 1; + } +} + int sj4_from_utf16(u_char* out, const u_char* in, int len) { const wchar_t* w_in = (const wchar_t*)in; const wchar_t* o_in = w_in; u_char* o_out = out; while((w_in - o_in) < len) { - u_int n = 0; + int n = 0; - if(is_high_surr(w_in[0])) { - if(is_low_urr(w_in[1])) { - n = 0x10000 + CAST_I32(w_in[0] - 0xd800) * 0x400 + CAST_I32(w_in[1] - 0xdc00); - - w_in += 2; - } else if(w_in[1] == 0) { - n = w_in[0]; - - w_in += 2; - } else { - return 0; - } - } else if(is_low_urr(w_in[0])) { - if(w_in[1] == 0) { - n = w_in[0]; - - w_in += 2; - } else { - return 0; - } - } else { - n = w_in[0]; - - w_in += 1; - } + w_in += sj4_utf16_read((const unsigned char*)w_in, &n); n = ucs_to_euc(n); @@ -192,35 +212,55 @@ int sj4_from_utf16(u_char* out, const u_char* in, int len) { return out - o_out; } +#endif -static u_int eucjp_read(const u_char* in) { +int sj4_eucjp_read(const unsigned char* in, int* output) { u_char c = *in; u_int n = 0; if(c <= 0x7f) { - n = c; + *output = c; + + return 1; } else if(c != 0x8f) { n = CAST_I32(in[0]) << 8; n |= CAST_I32(in[1]); + + *output = n; + + return 2; } else { n = CAST_I32(in[0]) << 16; n |= CAST_I32(in[1]) << 8; n |= CAST_I32(in[2]); - } - return n; -} + *output = n; -static int eucjp_codesize(u_int n) { - if(n <= 0xff) { - return 1; - } else if(n <= 0xffff) { - return 2; - } else { return 3; } } -#endif + +int sj4_sjis_read(const unsigned char* in, int* output) { + u_char c; + u_int n = 0; + + c = in[0]; + + if(c <= 0x7f || (0xa1 <= c && c <= 0xdf)) { + *output = c; + + return 1; + } else if((0x81 <= c && c <= 0x9f) || (0xe0 <= c && c <= 0xfc)) { + n = CAST_I32(in[0]) << 8; + n |= in[1]; + + *output = n; + + return 2; + } + + return 0; +} int sj4_to_sjis(u_char* out, const u_char* in, int len) { unsigned char d[] = {0, 0}; @@ -235,10 +275,10 @@ int sj4_to_utf8(u_char* out, const u_char* in, int len) { u_char* o_out = out; while((in - o_in) < len) { - int i, j = 0; - u_int n = eucjp_read(in); + int i, j = 0; + int n; - in += eucjp_codesize(n); + in += sj4_eucjp_read(in, &n); n = euc_to_ucs(n); @@ -272,10 +312,10 @@ int sj4_to_utf16(u_char* out, const u_char* in, int len) { wchar_t* o_out = w_out; while((in - o_in) < len) { - int i, j = 0; - u_int n = eucjp_read(in); + int i, j = 0; + int n; - in += eucjp_codesize(n); + in += sj4_eucjp_read(in, &n); n = euc_to_ucs(n); @@ -322,3 +362,18 @@ int sj4_charset_from(int type, void* output, const void* input, int len) { return len; } + +int sj4_charset_read(int type, int* output, const void* input) { + if(type == SJ4EUCJP) { + return sj4_eucjp_read(input, output); + } else if(type == SJ4SJIS) { + return sj4_sjis_read(input, output); + } +#ifdef UCS + else if(type == SJ4UTF8) { + return sj4_utf8_read(input, output); + } else if(type == SJ4UTF16) { + return sj4_utf16_read(input, output); + } +#endif +} diff --git a/lib/sj4core/charsize.c b/lib/sj4core/charsize.c index 07d8d06..c87505e 100644 --- a/lib/sj4core/charsize.c +++ b/lib/sj4core/charsize.c @@ -32,21 +32,9 @@ #include "sj_typedef.h" #include "sj_dict.h" +#define NO_CODESIZE_INLINE #include "sj_kanakan.h" int codesize(u_char code) { - switch(code & KANJIMODEMASK) { - case ZENHIRAASSYUKU: - case ZENKATAASSYUKU: - case KANJIASSYUKU: - case KANJISTREND: - return 1; - case LEADINGHANKAKU: - case OFFSETASSYUKU: - case AIATTRIBUTE: - case KANAKANCOST: - return 2; - default: - return 2; - } + return codesize_inline(code); } diff --git a/lib/sj4core/dict.c b/lib/sj4core/dict.c index f727333..e45c2df 100644 --- a/lib/sj4core/dict.c +++ b/lib/sj4core/dict.c @@ -60,6 +60,8 @@ void get_askknj(SJ4_CONTEXT2) { p++; askknj_k[i++] = r; } + + askbase = dicbuf; } int seldict(SJ4_CONTEXT TypeDicID id) { diff --git a/lib/sj4core/hzstrlen.c b/lib/sj4core/hzstrlen.c index 8a2289b..1224f71 100644 --- a/lib/sj4core/hzstrlen.c +++ b/lib/sj4core/hzstrlen.c @@ -33,18 +33,11 @@ #include "sj_const.h" #include "sj_dict.h" +#define NO_EUC_CODESIZE_INLINE #include "sj_kanakan.h" int euc_codesize(u_char c) { - if((c & KANJIMODEMASK) == 0x90) { - return 1; - } else if(isknj1(c)) { - return 2; - } else if(c == SS3) { - return 3; - } else { - return 1; - } + return euc_codesize_inline(c); } int hzstrlen(u_char* ptr, int len) { diff --git a/lib/sj4core/mkjiritu.c b/lib/sj4core/mkjiritu.c index 8bc3639..41f35f0 100644 --- a/lib/sj4core/mkjiritu.c +++ b/lib/sj4core/mkjiritu.c @@ -85,6 +85,8 @@ dic_mu(SJ4_CONTEXT int mode) { for(dp = dictlist; dp; dp = dp->next) { curdict = dp->dict; + (*curdict->getofs)(SJ4_CONTEXT_PASS curdict); + (*curdict->getidx)(SJ4_CONTEXT_PASS curdict); dicinl = 1; dicsaml = 0; prevseg = -1; diff --git a/lib/sj4core/mkkouho.c b/lib/sj4core/mkkouho.c index 4820edc..84dac10 100644 --- a/lib/sj4core/mkkouho.c +++ b/lib/sj4core/mkkouho.c @@ -35,14 +35,20 @@ #include "sj_kanakan.h" -static void getkhtbl(SJ4_CONTEXT CLREC*), cl_kanji(SJ4_CONTEXT JREC*, CLREC*); -static int diffknj(SJ4_CONTEXT JREC*, u_char*, int); +static void getkhtbl(SJ4_CONTEXT CLREC*, u_char (*)[MAXWDKANJILEN], int*); +static void cl_kanji(SJ4_CONTEXT JREC*, CLREC*, u_char (*)[MAXWDKANJILEN], int*); +static int diffknj(SJ4_CONTEXT JREC*, u_char*, int, u_char (*)[MAXWDKANJILEN], int*); static void cl_numcmn(SJ4_CONTEXT JREC*, CLREC*); static u_char* makekan(SJ4_CONTEXT u_char*, u_char*, int); void mkkouho(SJ4_CONTEXT2) { CLREC* clrec; int keeplen; + u_char kcache[64][MAXWDKANJILEN]; + int kvalid[64]; + int i; + + for(i = 0; i < 64; i++) kvalid[i] = -1; khcount = nkhcount = 0; @@ -51,20 +57,20 @@ void mkkouho(SJ4_CONTEXT2) { clrec = clt1st; do { - getkhtbl(SJ4_CONTEXT_PASS clrec); + getkhtbl(SJ4_CONTEXT_PASS clrec, kcache, kvalid); clrec = clrec->clsort; } while(clrec && (keeplen == clrec->cllen)); } static void -getkhtbl(SJ4_CONTEXT CLREC* clrec) { +getkhtbl(SJ4_CONTEXT CLREC* clrec, u_char kcache[][MAXWDKANJILEN], int* kvalid) { JREC* jrec; jrec = clrec->jnode; switch(jrec->jclass) { case C_DICT: - cl_kanji(SJ4_CONTEXT_PASS jrec, clrec); + cl_kanji(SJ4_CONTEXT_PASS jrec, clrec, kcache, kvalid); break; case C_N_ARABIA: case C_N_ARACMA: @@ -88,7 +94,7 @@ getkhtbl(SJ4_CONTEXT CLREC* clrec) { } static void -cl_kanji(SJ4_CONTEXT JREC* jrec, CLREC* clrec) { +cl_kanji(SJ4_CONTEXT JREC* jrec, CLREC* clrec, u_char kcache[][MAXWDKANJILEN], int* kvalid) { u_char* ptr; int kcount = khcount; @@ -99,7 +105,7 @@ cl_kanji(SJ4_CONTEXT JREC* jrec, CLREC* clrec) { get_askknj(SJ4_CONTEXT_PASS2); while(*ptr != HINSIBLKTERM) { - if(diffknj(SJ4_CONTEXT_PASS jrec, ptr, kcount)) { + if(diffknj(SJ4_CONTEXT_PASS jrec, ptr, kcount, kcache, kvalid)) { setkouho(SJ4_CONTEXT_PASS clrec, (TypeDicOfs)(ptr - dicbuf), 0); } ptr = skipkstr(ptr); @@ -233,7 +239,7 @@ makekan(SJ4_CONTEXT u_char* s, u_char* d, int flg) { default: d = makekan_norm(SJ4_CONTEXT_PASS s, d, FALSE); } - s += codesize(*s); + s += csize; } return d; @@ -300,12 +306,11 @@ sameknj(u_char* p, u_char plen, u_char* q, u_char qlen) { #undef QEND static int -diffknj(SJ4_CONTEXT JREC* jrec, u_char* ptr, int num) { +diffknj(SJ4_CONTEXT JREC* jrec, u_char* ptr, int num, u_char kcache[][MAXWDKANJILEN], int* kvalid) { KHREC* kptr; JREC* jptr; int i; u_char kbuf1[MAXWDKANJILEN]; - u_char kbuf2[MAXWDKANJILEN]; if(jrec->hinsi == TANKANJI) return TRUE; @@ -327,10 +332,17 @@ diffknj(SJ4_CONTEXT JREC* jrec, u_char* ptr, int num) { if(jptr->stbofs != jrec->stbofs) continue; - makekan(SJ4_CONTEXT_PASS dicbuf + kptr->offs, kbuf2, TRUE); + { + int slot = i & (64 - 1); - if(sameknj(kbuf1, jrec->jlen, kbuf2, jptr->jlen)) - return FALSE; + if(kvalid[slot] != i) { + makekan(SJ4_CONTEXT_PASS dicbuf + kptr->offs, kcache[slot], TRUE); + kvalid[slot] = i; + } + + if(sameknj(kbuf1, jrec->jlen, kcache[slot], jptr->jlen)) + return FALSE; + } } return TRUE; diff --git a/src/sj4test/main.c b/src/sj4test/main.c index 43c8aea..4ccc3b4 100644 --- a/src/sj4test/main.c +++ b/src/sj4test/main.c @@ -1,4 +1,4 @@ -#define UTF16 +// #define UTF16 #include #include #include diff --git a/src/sj4test2/main.c b/src/sj4test2/main.c index 29e5eb7..65eebe4 100644 --- a/src/sj4test2/main.c +++ b/src/sj4test2/main.c @@ -14,7 +14,15 @@ #endif static void ime_packet(Sj4Ime* ime, int type, void* data) { - printf("%d %p\n", type, data); + if(type == Sj4ImeBeginKanList) { + printf("\nbegin kan list\n"); + } else if(type == Sj4ImeKanEntry) { + printf(" %s\n", data); + } else if(type == Sj4ImeEndKanList) { + printf("end kan list\n"); + } else { + printf("%d %p\n", type, data); + } } int main() {