pyrite-dev/sj4
12
2
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add KANAKANCOST

Browse source
This commit is contained in:
Nishi 2026-06-21 08:00:28 +09:00
commit a6a7858eaf
20 changed files with 58916 additions and 59081 deletions
Download patch file Download diff file Expand all files Collapse all files

117868
dict/visual.plus.dic
View file

File diff suppressed because it is too large Load diff

1
include/sj4core/sj_dict.h
View file

@ -56,6 +56,7 @@
#define AIATTRIBUTE 0xb0
#define KANJIASSYUKU 0xc0
#define LEADINGHANKAKU 0xd0
#define KANAKANCOST 0xe0
#define KANJISTREND 0x00
#define ISASSYUKU 0x80
#define ASSUKUUSE2BYTE 0x90

5
include/sj4core/sj_kanakan.h
View file

@ -118,10 +118,7 @@ int seldict(SJ4_CONTEXT TypeDicID);
u_char* get_idxptr(SJ4_CONTEXT TypeDicSeg);
/* functbl.c */
extern VFunc cvtnum_func[];
extern UCPFunc getkan_func[];
extern IFunc setjrec_func[];
extern IFunc hiraknj_func[];
extern VFunc cvtnum_func[];
/* fuzoku.c */
void setclrec(SJ4_CONTEXT JREC*, u_char*, TypeCnct);

4
include/sj4core/sj_rename.h
View file

@ -405,10 +405,6 @@
#define rigtadr Jrigtadr
#define cvtnum_func Jcvtnum_func
#define getkan_func Jgetkan_func
#define setjrec_func Jsetjrec_func
#define srchg_func Jsrchg_func
#define hiraknj_func Jhiraknj_func
#define fzkadr Jfzkadr

1
include/sj4core/sj_struct.h
View file

@ -51,6 +51,7 @@ typedef struct jiritu {
u_char stbofs;
u_char count;
u_char numlen;
u_short cost; /* new in SJ4 */
} JREC;
typedef struct bunsetu {

5
include/sj4core/sj_table.h
View file

@ -79,12 +79,9 @@
#define Connadr connadr
#define Cvtnum_func cvtnum_func
#define Fzkadr(i) fzkadr[i]
#define Getkan_func getkan_func
#define Hiraknj_func hiraknj_func
#define Kigou kigou
#define Kurai1_tbl kurai1_tbl
#define Kurai2_tbl kurai2_tbl
#define Makekan_func makekan_func
#define Num1tbl num1tbl
#define Num2tbl num2tbl
#define Num3tbl num3tbl
@ -94,9 +91,7 @@
#define Rigtadr(i) rigtadr[i]
#define Scncttbl(i, j) scncttbl[i][j]
#define Selsjadrs(i) selsjadrs[i]
#define Setjrec_func setjrec_func
#define Settou_ptr(i) settou_ptr[i]
#define Srchg_func srchg_func
#define Stbadr(i) stbadr[i]
#define Sttpri(i, j) sttpri[i][j]
#define Suuji_tbl suuji_tbl

1
lib/sj4core/charsize.c
View file

@ -50,6 +50,7 @@ int codesize(u_char code) {
case LEADINGHANKAKU:
case OFFSETASSYUKU:
case AIATTRIBUTE:
case KANAKANCOST:
return 2;
default:
return 2;

24
lib/sj4core/functbl.c
View file

@ -43,27 +43,3 @@ VFunc cvtnum_func[] = {
(VFunc)num_type04, (VFunc)num_type05, (VFunc)num_type06, (VFunc)num_type07,
(VFunc)num_type08, (VFunc)num_type09, (VFunc)num_type10, (VFunc)num_type11,
(VFunc)num_type12, (VFunc)num_type13, (VFunc)num_type14};
UCPFunc getkan_func[] = {
(UCPFunc)getkan_norm, (UCPFunc)getkan_norm, (UCPFunc)getkan_hira, (UCPFunc)getkan_kata,
(UCPFunc)getkan_knj, (UCPFunc)getkan_ofs, (UCPFunc)getkan_norm, (UCPFunc)getkan_none,
(UCPFunc)getkan_norm, (UCPFunc)getkan_norm, (UCPFunc)getkan_hira, (UCPFunc)getkan_kata,
(UCPFunc)getkan_knj, (UCPFunc)getkan_ofs, (UCPFunc)getkan_norm, (UCPFunc)getkan_none};
IFunc setjrec_func[] = {
(IFunc)setj_norm2, (IFunc)setj_norm2, (IFunc)setj_norm1, (IFunc)setj_norm1,
(IFunc)setj_knj, (IFunc)setj_ofs, (IFunc)setj_norm2, (IFunc)setj_atrb,
(IFunc)setj_norm2, (IFunc)setj_norm2, (IFunc)setj_norm1, (IFunc)setj_norm1,
(IFunc)setj_knj, (IFunc)setj_ofs, (IFunc)setj_norm2, (IFunc)setj_atrb};
IFunc hiraknj_func[] = {
(IFunc)hiraknj_hira, (IFunc)hiraknj_norm, (IFunc)hiraknj_hask, (IFunc)hiraknj_kask,
(IFunc)hiraknj_knj, (IFunc)hiraknj_ofs, (IFunc)hiraknj_norm, (IFunc)hiraknj_atrb,
(IFunc)hiraknj_hira, (IFunc)hiraknj_norm, (IFunc)hiraknj_hask, (IFunc)hiraknj_kask,
(IFunc)hiraknj_knj, (IFunc)hiraknj_ofs, (IFunc)hiraknj_norm, (IFunc)hiraknj_atrb};
UCPFunc makekan_func[] = {
(UCPFunc)makekan_norm, (UCPFunc)makekan_norm, (UCPFunc)makekan_1byte, (UCPFunc)makekan_1byte,
(UCPFunc)makekan_knj, (UCPFunc)makekan_ofs, (UCPFunc)makekan_norm, (UCPFunc)makekan_none,
(UCPFunc)makekan_norm, (UCPFunc)makekan_norm, (UCPFunc)makekan_1byte, (UCPFunc)makekan_1byte,
(UCPFunc)makekan_knj, (UCPFunc)makekan_ofs, (UCPFunc)makekan_norm, (UCPFunc)makekan_none};

2
lib/sj4core/getkanji.c
View file

@ -151,6 +151,7 @@ code2kanji(SJ4_CONTEXT u_char* s, u_char* d, u_char* ym, int yl, int flg) {
break;
case AIATTRIBUTE:
case KANAKANCOST:
d = getkan_none(s, d, ym, yl, flg);
break;
@ -181,6 +182,7 @@ code2kanji(SJ4_CONTEXT u_char* s, u_char* d, u_char* ym, int yl, int flg) {
break;
case AIATTRIBUTE:
case KANAKANCOST:
d = getkan_none(s, d, ym, yl, FALSE);
break;

12
lib/sj4core/mk2claus.c
View file

@ -60,6 +60,16 @@ set2nd(SJ4_CONTEXT CLREC* clrec) {
return rec2 ? (clrec->cllen + rec2->cllen) : (clrec->cllen);
}
static int cost_penalty(CLREC* clrec) {
int c = clrec->jnode->cost;
if(c < 0xfff / 5 * 1) return 8;
if(c < 0xfff / 5 * 2) return 4;
if(c < 0xfff / 5 * 3) return 2;
if(c < 0xfff / 5 * 4) return 1;
return 0;
};
void mk2claus(SJ4_CONTEXT2) {
u_char* keepptr;
int keeplen;
@ -117,7 +127,7 @@ void mk2claus(SJ4_CONTEXT2) {
if(selcl->cl2len > clrec->cl2len) continue;
prty = priority(SJ4_CONTEXT_PASS clrec) - prty2;
prty = priority(SJ4_CONTEXT_PASS clrec) - prty2 + cost_penalty(clrec);
prty2++;

2
lib/sj4core/mkkouho.c
View file

@ -193,6 +193,7 @@ makekan(SJ4_CONTEXT u_char* s, u_char* d, int flg) {
break;
case AIATTRIBUTE:
case KANAKANCOST:
d = makekan_none(SJ4_CONTEXT_PASS s, d, flg);
break;
@ -223,6 +224,7 @@ makekan(SJ4_CONTEXT u_char* s, u_char* d, int flg) {
break;
case AIATTRIBUTE:
case KANAKANCOST:
d = makekan_none(SJ4_CONTEXT_PASS s, d, FALSE);
break;

19
lib/sj4core/setjrec.c
View file

@ -79,6 +79,10 @@ int setj_ofs(SJ4_CONTEXT u_char* p) {
contf = FALSE;
break;
case KANAKANCOST:
p += 2;
break;
default:
p += setj_norm2(SJ4_CONTEXT_PASS p);
}
@ -123,6 +127,10 @@ int setj_knj(SJ4_CONTEXT u_char* p) {
contf = FALSE;
break;
case KANAKANCOST:
p += 2;
break;
default:
p += setj_norm2(SJ4_CONTEXT_PASS p);
}
@ -171,6 +179,7 @@ void setjrec(SJ4_CONTEXT u_char* tagp, int mode) {
int plen;
int nlen;
int len;
int cost = 0;
plen = getplen(tagp);
nlen = getnlen(tagp);
@ -214,6 +223,11 @@ void setjrec(SJ4_CONTEXT u_char* tagp, int mode) {
tmp++;
break;
case KANAKANCOST:
cost = ((*tmp & 0xf) << 8) | *(tmp + 1);
tmp += 2;
break;
default:
tmp += setj_norm2(SJ4_CONTEXT_PASS tmp);
}
@ -232,6 +246,7 @@ void setjrec(SJ4_CONTEXT u_char* tagp, int mode) {
rec->dicid = curdict->dicid;
rec->hinsi = gram;
rec->sttofs = headcode;
rec->cost = cost;
}
}
@ -282,6 +297,10 @@ void setnumrec(SJ4_CONTEXT u_char* tagp, JREC* rec, TypeGram gram) {
tmp++;
break;
case KANAKANCOST:
tmp += 2;
break;
default:
tmp += setj_norm2(SJ4_CONTEXT_PASS tmp);
}

5
lib/sj4core/setkouho.c
View file

@ -186,6 +186,11 @@ hiraknj(SJ4_CONTEXT u_char* p) {
result = hiraknj_knj(SJ4_CONTEXT_PASS p, &i);
break;
case KANAKANCOST:
result = 0;
i = 2;
break;
default:
result = hiraknj_hira(SJ4_CONTEXT_PASS p, &i);
}

12
src/sj4mkdic/cnvhinsi.c
View file

@ -62,6 +62,18 @@ int u_strcmp(u_char* a, u_char* b) {
int cnvhinsi(u_char* buf) {
int i;
if(buf[0] == 'C') {
int n = atoi(buf + 1);
if(n < 0) n = 0;
if(n > 0xffff) n = 0xffff;
n = n * 0xfff / 0xffff;
return -(KANAKANCOST_BIT | n);
}
#if 1
for(i = 0; i <= GramMax; i++) {
if(u_strcmp(buf, (u_char*)gramtbl[i].name) == 0) return gramtbl[i].code;

11
src/sj4mkdic/makelist.c
View file

@ -107,10 +107,17 @@ makekanji(int* yomi, int* kanji, int* atr, int* len) {
while(*atr) {
i = *atr;
atr++;
if(i & KANAKANCOST_BIT) {
i &= ~KANAKANCOST_BIT;
ktmp[pos++] = (KANAKANCOST | ((i >> 8) & 0xff));
ktmp[pos++] = (i & 0xff);
} else {
#ifndef NO_ATR
ktmp[pos++] = (AIATTRIBUTE | ((i >> 8) & 0xff));
ktmp[pos++] = (i & 0xff);
ktmp[pos++] = (AIATTRIBUTE | ((i >> 8) & 0xff));
ktmp[pos++] = (i & 0xff);
#endif
}
}
if((i = top_strcmp(yomi, kanji))) {

3
src/sj4mkdic/readline.c
View file

@ -215,7 +215,8 @@ retry:
if(hinsi[0] == '[' && hinsi[i - 1] == ']') {
hinsi[i - 1] = 0;
i = cnvhinsi(hinsi + 1);
i = cnvhinsi(hinsi + 1);
if(i > 0) {
fprintf(stderr,
#ifdef ENGLISH

3
src/sj4mkdic/sj4_dict_const.h
View file

@ -69,6 +69,7 @@
#define AIATTRIBUTE 0xb0
#define KANJIASSYUKU 0xc0
#define LEADINGHANKAKU 0xd0
#define KANAKANCOST 0xe0
#define KANJISTREND 0x00
#define ISASSYUKU 0x80
#define ASSUKUUSE2BYTE 0x90
@ -168,4 +169,6 @@
#define HINSIREC 7
#define DOUONREC 8
#define KANAKANCOST_BIT (1 << 24)
#endif

0
format.sh → tools/format.sh
View file

0
win32.sh → tools/win32.sh
View file

19
wnn2sj4.py → tools/wnn2sj4.py
View file

@ -1,6 +1,10 @@
#!/usr/bin/env python3
# THIS IS FOR SYSTEM DICTIONARY GENERATION!!!!!
import sys
import re
from math import *
from collections import defaultdict
HINSHI = {
@ -118,6 +122,9 @@ for lineno, line in enumerate(fp, 1):
if h in SKIP:
continue
if re.findall(r'[^あ-んー]', y):
continue
sj = HINSHI[h]
if sj is None:
@ -136,8 +143,13 @@ fp.close()
grouped = defaultdict(list)
maxf = 0
for (y, k, sj), freq in entries.items():
grouped[y].append((freq, k, sj))
if maxf < freq:
maxf = freq
grouped[y].append((freq, k, sj, freq))
fp = open(outp, "w", encoding="utf-8")
for y in sorted(grouped.keys()):
@ -147,7 +159,8 @@ for y in sorted(grouped.keys()):
t = "\t" * (4 - (len(y) // 4))
for freq, k, sj in lst:
for freq, k, sj, freq in lst:
t2 = "\t" * (4 - (len(k) // 4))
fp.write(f"{y}{t}{k}{t2}{sj}:\n")
freq = floor(freq / maxf * 0xffff)
fp.write(f"{y}{t}{k}{t2}{sj}:[C{freq}]:\n")
fp.close()