pyrite-dev/sj4
12
2
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

several optimization

Browse source
This commit is contained in:
Nishi 2026-09-23 13:24:26 +09:00
commit c52b21c9ee
12 changed files with 214 additions and 111 deletions
Download patch file Download diff file Expand all files Collapse all files

211
lib/sj4charset/conv.c
View file

@ -94,46 +94,60 @@ static int utf8_later(unsigned char c) {
#define CAST_I32(x) ((int)(x))
int sj4_utf8_read(const unsigned char* in, int* output) {
u_char c = in[0];
u_int n = 0;
if(c < 0x80) {
*output = c;
return 1;
} else if(0xc2 <= c && c < 0xe0) {
if(!utf8_later(in[1])) return 0;
if((in[0] & 0x1e) == 0) return 0;
n = CAST_I32(in[0] & 0x1f) << 6;
n |= CAST_I32(in[1] & 0x3f);
*output = n;
return 2;
} else if(0xe0 <= c && c < 0xf0) {
if(!utf8_later(in[1]) || !utf8_later(in[2])) return 0;
if((in[0] & 0x0f) == 0 && (in[1] & 0x20) == 0) return 0;
n = CAST_I32(in[0] & 0x0f) << 12;
n |= CAST_I32(in[1] & 0x3f) << 6;
n |= CAST_I32(in[2] & 0x3f);
*output = n;
return 3;
} else if(0xf0 <= c && c < 0xf8) {
if(!utf8_later(in[1]) || !utf8_later(in[2]) || !utf8_later(in[3])) return 0;
if((in[0] & 0x07) == 0 && (in[1] & 0x30) == 0) return 0;
n = CAST_I32(in[0] & 0x07) << 18;
n |= CAST_I32(in[1] & 0x3f) << 12;
n |= CAST_I32(in[2] & 0x3f) << 6;
n |= CAST_I32(in[3] & 0x3f);
*output = n;
return 4;
}
return 0;
}
int sj4_from_utf8(u_char* out, const u_char* in, int len) {
const u_char* o_in = in;
u_char* o_out = out;
while((in - o_in) < len) {
u_char c = *in;
u_int n = 0;
int n;
if(c < 0x80) {
n = c;
in += 1;
} else if(0xc2 <= c && c < 0xe0) {
if(!utf8_later(in[1])) return 0;
if((in[0] & 0x1e) == 0) return 0;
n = CAST_I32(in[0] & 0x1f) << 6;
n |= CAST_I32(in[1] & 0x3f);
in += 2;
} else if(0xe0 <= c && c < 0xf0) {
if(!utf8_later(in[1]) || !utf8_later(in[2])) return 0;
if((in[0] & 0x0f) == 0 && (in[1] & 0x20) == 0) return 0;
n = CAST_I32(in[0] & 0x0f) << 12;
n |= CAST_I32(in[1] & 0x3f) << 6;
n |= CAST_I32(in[2] & 0x3f);
in += 3;
} else if(0xf0 <= c && c < 0xf8) {
if(!utf8_later(in[1]) || !utf8_later(in[2]) || !utf8_later(in[3])) return 0;
if((in[0] & 0x07) == 0 && (in[1] & 0x30) == 0) return 0;
n = CAST_I32(in[0] & 0x07) << 18;
n |= CAST_I32(in[1] & 0x3f) << 12;
n |= CAST_I32(in[2] & 0x3f) << 6;
n |= CAST_I32(in[3] & 0x3f);
in += 4;
}
in += sj4_utf8_read(in, &n);
n = ucs_to_euc(n);
@ -151,39 +165,45 @@ static int is_low_urr(wchar_t ch) {
return 0xDC00 <= ch && ch < 0xE000;
}
int sj4_utf16_read(const unsigned char* in, int* output) {
const wchar_t* w_in = (const wchar_t*)in;
if(is_high_surr(w_in[0])) {
if(is_low_urr(w_in[1])) {
*output = 0x10000 + CAST_I32(w_in[0] - 0xd800) * 0x400 + CAST_I32(w_in[1] - 0xdc00);
return 2;
} else if(w_in[1] == 0) {
*output = w_in[0];
return 2;
} else {
return 0;
}
} else if(is_low_urr(w_in[0])) {
if(w_in[1] == 0) {
*output = w_in[0];
return 2;
} else {
return 0;
}
} else {
*output = w_in[0];
return 1;
}
}
int sj4_from_utf16(u_char* out, const u_char* in, int len) {
const wchar_t* w_in = (const wchar_t*)in;
const wchar_t* o_in = w_in;
u_char* o_out = out;
while((w_in - o_in) < len) {
u_int n = 0;
int n = 0;
if(is_high_surr(w_in[0])) {
if(is_low_urr(w_in[1])) {
n = 0x10000 + CAST_I32(w_in[0] - 0xd800) * 0x400 + CAST_I32(w_in[1] - 0xdc00);
w_in += 2;
} else if(w_in[1] == 0) {
n = w_in[0];
w_in += 2;
} else {
return 0;
}
} else if(is_low_urr(w_in[0])) {
if(w_in[1] == 0) {
n = w_in[0];
w_in += 2;
} else {
return 0;
}
} else {
n = w_in[0];
w_in += 1;
}
w_in += sj4_utf16_read((const unsigned char*)w_in, &n);
n = ucs_to_euc(n);
@ -192,35 +212,55 @@ int sj4_from_utf16(u_char* out, const u_char* in, int len) {
return out - o_out;
}
#endif
static u_int eucjp_read(const u_char* in) {
int sj4_eucjp_read(const unsigned char* in, int* output) {
u_char c = *in;
u_int n = 0;
if(c <= 0x7f) {
n = c;
*output = c;
return 1;
} else if(c != 0x8f) {
n = CAST_I32(in[0]) << 8;
n |= CAST_I32(in[1]);
*output = n;
return 2;
} else {
n = CAST_I32(in[0]) << 16;
n |= CAST_I32(in[1]) << 8;
n |= CAST_I32(in[2]);
}
return n;
}
*output = n;
static int eucjp_codesize(u_int n) {
if(n <= 0xff) {
return 1;
} else if(n <= 0xffff) {
return 2;
} else {
return 3;
}
}
#endif
int sj4_sjis_read(const unsigned char* in, int* output) {
u_char c;
u_int n = 0;
c = in[0];
if(c <= 0x7f || (0xa1 <= c && c <= 0xdf)) {
*output = c;
return 1;
} else if((0x81 <= c && c <= 0x9f) || (0xe0 <= c && c <= 0xfc)) {
n = CAST_I32(in[0]) << 8;
n |= in[1];
*output = n;
return 2;
}
return 0;
}
int sj4_to_sjis(u_char* out, const u_char* in, int len) {
unsigned char d[] = {0, 0};
@ -235,10 +275,10 @@ int sj4_to_utf8(u_char* out, const u_char* in, int len) {
u_char* o_out = out;
while((in - o_in) < len) {
int i, j = 0;
u_int n = eucjp_read(in);
int i, j = 0;
int n;
in += eucjp_codesize(n);
in += sj4_eucjp_read(in, &n);
n = euc_to_ucs(n);
@ -272,10 +312,10 @@ int sj4_to_utf16(u_char* out, const u_char* in, int len) {
wchar_t* o_out = w_out;
while((in - o_in) < len) {
int i, j = 0;
u_int n = eucjp_read(in);
int i, j = 0;
int n;
in += eucjp_codesize(n);
in += sj4_eucjp_read(in, &n);
n = euc_to_ucs(n);
@ -322,3 +362,18 @@ int sj4_charset_from(int type, void* output, const void* input, int len) {
return len;
}
int sj4_charset_read(int type, int* output, const void* input) {
if(type == SJ4EUCJP) {
return sj4_eucjp_read(input, output);
} else if(type == SJ4SJIS) {
return sj4_sjis_read(input, output);
}
#ifdef UCS
else if(type == SJ4UTF8) {
return sj4_utf8_read(input, output);
} else if(type == SJ4UTF16) {
return sj4_utf16_read(input, output);
}
#endif
}

16
lib/sj4core/charsize.c
View file

@ -32,21 +32,9 @@
#include "sj_typedef.h"
#include "sj_dict.h"
#define NO_CODESIZE_INLINE
#include "sj_kanakan.h"
int codesize(u_char code) {
switch(code & KANJIMODEMASK) {
case ZENHIRAASSYUKU:
case ZENKATAASSYUKU:
case KANJIASSYUKU:
case KANJISTREND:
return 1;
case LEADINGHANKAKU:
case OFFSETASSYUKU:
case AIATTRIBUTE:
case KANAKANCOST:
return 2;
default:
return 2;
}
return codesize_inline(code);
}

2
lib/sj4core/dict.c
View file

@ -60,6 +60,8 @@ void get_askknj(SJ4_CONTEXT2) {
p++;
askknj_k[i++] = r;
}
askbase = dicbuf;
}
int seldict(SJ4_CONTEXT TypeDicID id) {

11
lib/sj4core/hzstrlen.c
View file

@ -33,18 +33,11 @@
#include "sj_const.h"
#include "sj_dict.h"
#define NO_EUC_CODESIZE_INLINE
#include "sj_kanakan.h"
int euc_codesize(u_char c) {
if((c & KANJIMODEMASK) == 0x90) {
return 1;
} else if(isknj1(c)) {
return 2;
} else if(c == SS3) {
return 3;
} else {
return 1;
}
return euc_codesize_inline(c);
}
int hzstrlen(u_char* ptr, int len) {

2
lib/sj4core/mkjiritu.c
View file

@ -85,6 +85,8 @@ dic_mu(SJ4_CONTEXT int mode) {
for(dp = dictlist; dp; dp = dp->next) {
curdict = dp->dict;
(*curdict->getofs)(SJ4_CONTEXT_PASS curdict);
(*curdict->getidx)(SJ4_CONTEXT_PASS curdict);
dicinl = 1;
dicsaml = 0;
prevseg = -1;

38
lib/sj4core/mkkouho.c
View file

@ -35,14 +35,20 @@
#include "sj_kanakan.h"
static void getkhtbl(SJ4_CONTEXT CLREC*), cl_kanji(SJ4_CONTEXT JREC*, CLREC*);
static int diffknj(SJ4_CONTEXT JREC*, u_char*, int);
static void getkhtbl(SJ4_CONTEXT CLREC*, u_char (*)[MAXWDKANJILEN], int*);
static void cl_kanji(SJ4_CONTEXT JREC*, CLREC*, u_char (*)[MAXWDKANJILEN], int*);
static int diffknj(SJ4_CONTEXT JREC*, u_char*, int, u_char (*)[MAXWDKANJILEN], int*);
static void cl_numcmn(SJ4_CONTEXT JREC*, CLREC*);
static u_char* makekan(SJ4_CONTEXT u_char*, u_char*, int);
void mkkouho(SJ4_CONTEXT2) {
CLREC* clrec;
int keeplen;
u_char kcache[64][MAXWDKANJILEN];
int kvalid[64];
int i;
for(i = 0; i < 64; i++) kvalid[i] = -1;
khcount = nkhcount = 0;
@ -51,20 +57,20 @@ void mkkouho(SJ4_CONTEXT2) {
clrec = clt1st;
do {
getkhtbl(SJ4_CONTEXT_PASS clrec);
getkhtbl(SJ4_CONTEXT_PASS clrec, kcache, kvalid);
clrec = clrec->clsort;
} while(clrec && (keeplen == clrec->cllen));
}
static void
getkhtbl(SJ4_CONTEXT CLREC* clrec) {
getkhtbl(SJ4_CONTEXT CLREC* clrec, u_char kcache[][MAXWDKANJILEN], int* kvalid) {
JREC* jrec;
jrec = clrec->jnode;
switch(jrec->jclass) {
case C_DICT:
cl_kanji(SJ4_CONTEXT_PASS jrec, clrec);
cl_kanji(SJ4_CONTEXT_PASS jrec, clrec, kcache, kvalid);
break;
case C_N_ARABIA:
case C_N_ARACMA:
@ -88,7 +94,7 @@ getkhtbl(SJ4_CONTEXT CLREC* clrec) {
}
static void
cl_kanji(SJ4_CONTEXT JREC* jrec, CLREC* clrec) {
cl_kanji(SJ4_CONTEXT JREC* jrec, CLREC* clrec, u_char kcache[][MAXWDKANJILEN], int* kvalid) {
u_char* ptr;
int kcount = khcount;
@ -99,7 +105,7 @@ cl_kanji(SJ4_CONTEXT JREC* jrec, CLREC* clrec) {
get_askknj(SJ4_CONTEXT_PASS2);
while(*ptr != HINSIBLKTERM) {
if(diffknj(SJ4_CONTEXT_PASS jrec, ptr, kcount)) {
if(diffknj(SJ4_CONTEXT_PASS jrec, ptr, kcount, kcache, kvalid)) {
setkouho(SJ4_CONTEXT_PASS clrec, (TypeDicOfs)(ptr - dicbuf), 0);
}
ptr = skipkstr(ptr);
@ -233,7 +239,7 @@ makekan(SJ4_CONTEXT u_char* s, u_char* d, int flg) {
default:
d = makekan_norm(SJ4_CONTEXT_PASS s, d, FALSE);
}
s += codesize(*s);
s += csize;
}
return d;
@ -300,12 +306,11 @@ sameknj(u_char* p, u_char plen, u_char* q, u_char qlen) {
#undef QEND
static int
diffknj(SJ4_CONTEXT JREC* jrec, u_char* ptr, int num) {
diffknj(SJ4_CONTEXT JREC* jrec, u_char* ptr, int num, u_char kcache[][MAXWDKANJILEN], int* kvalid) {
KHREC* kptr;
JREC* jptr;
int i;
u_char kbuf1[MAXWDKANJILEN];
u_char kbuf2[MAXWDKANJILEN];
if(jrec->hinsi == TANKANJI) return TRUE;
@ -327,10 +332,17 @@ diffknj(SJ4_CONTEXT JREC* jrec, u_char* ptr, int num) {
if(jptr->stbofs != jrec->stbofs) continue;
makekan(SJ4_CONTEXT_PASS dicbuf + kptr->offs, kbuf2, TRUE);
{
int slot = i & (64 - 1);
if(sameknj(kbuf1, jrec->jlen, kbuf2, jptr->jlen))
return FALSE;
if(kvalid[slot] != i) {
makekan(SJ4_CONTEXT_PASS dicbuf + kptr->offs, kcache[slot], TRUE);
kvalid[slot] = i;
}
if(sameknj(kbuf1, jrec->jlen, kcache[slot], jptr->jlen))
return FALSE;
}
}
return TRUE;