Add KANAKANCOST

include/sj4core/sj_dict.h

@@ -56,6 +56,7 @@
 #define AIATTRIBUTE 0xb0
 #define KANJIASSYUKU 0xc0
 #define LEADINGHANKAKU 0xd0
+#define KANAKANCOST 0xe0
 #define KANJISTREND 0x00
 #define ISASSYUKU 0x80
 #define ASSUKUUSE2BYTE 0x90

include/sj4core/sj_kanakan.h

@@ -119,9 +119,6 @@ u_char* get_idxptr(SJ4_CONTEXT TypeDicSeg);
 
 /* functbl.c */
 extern VFunc cvtnum_func[];
-extern UCPFunc getkan_func[];
-extern IFunc   setjrec_func[];
-extern IFunc   hiraknj_func[];
 
 /* fuzoku.c */
 void	    setclrec(SJ4_CONTEXT JREC*, u_char*, TypeCnct);

include/sj4core/sj_rename.h

@@ -405,10 +405,6 @@
 #define rigtadr Jrigtadr
 
 #define cvtnum_func Jcvtnum_func
-#define getkan_func Jgetkan_func
-#define setjrec_func Jsetjrec_func
-#define srchg_func Jsrchg_func
-#define hiraknj_func Jhiraknj_func
 
 #define fzkadr Jfzkadr
 

include/sj4core/sj_struct.h

@@ -51,6 +51,7 @@ typedef struct jiritu {
 	u_char	  stbofs;
 	u_char	  count;
 	u_char	  numlen;
+	u_short	  cost; /* new in SJ4 */
 } JREC;
 
 typedef struct bunsetu {

include/sj4core/sj_table.h

@@ -79,12 +79,9 @@
 #define Connadr connadr
 #define Cvtnum_func cvtnum_func
 #define Fzkadr(i) fzkadr[i]
-#define Getkan_func getkan_func
-#define Hiraknj_func hiraknj_func
 #define Kigou kigou
 #define Kurai1_tbl kurai1_tbl
 #define Kurai2_tbl kurai2_tbl
-#define Makekan_func makekan_func
 #define Num1tbl num1tbl
 #define Num2tbl num2tbl
 #define Num3tbl num3tbl
@@ -94,9 +91,7 @@
 #define Rigtadr(i) rigtadr[i]
 #define Scncttbl(i, j) scncttbl[i][j]
 #define Selsjadrs(i) selsjadrs[i]
-#define Setjrec_func setjrec_func
 #define Settou_ptr(i) settou_ptr[i]
-#define Srchg_func srchg_func
 #define Stbadr(i) stbadr[i]
 #define Sttpri(i, j) sttpri[i][j]
 #define Suuji_tbl suuji_tbl

lib/sj4core/charsize.c

@@ -50,6 +50,7 @@ int codesize(u_char code) {
 	case LEADINGHANKAKU:
 	case OFFSETASSYUKU:
 	case AIATTRIBUTE:
+	case KANAKANCOST:
 		return 2;
 	default:
 		return 2;

lib/sj4core/functbl.c

@@ -43,27 +43,3 @@ VFunc cvtnum_func[] = {
     (VFunc)num_type04, (VFunc)num_type05, (VFunc)num_type06, (VFunc)num_type07,
     (VFunc)num_type08, (VFunc)num_type09, (VFunc)num_type10, (VFunc)num_type11,
     (VFunc)num_type12, (VFunc)num_type13, (VFunc)num_type14};
-
-UCPFunc getkan_func[] = {
-    (UCPFunc)getkan_norm, (UCPFunc)getkan_norm, (UCPFunc)getkan_hira, (UCPFunc)getkan_kata,
-    (UCPFunc)getkan_knj, (UCPFunc)getkan_ofs, (UCPFunc)getkan_norm, (UCPFunc)getkan_none,
-    (UCPFunc)getkan_norm, (UCPFunc)getkan_norm, (UCPFunc)getkan_hira, (UCPFunc)getkan_kata,
-    (UCPFunc)getkan_knj, (UCPFunc)getkan_ofs, (UCPFunc)getkan_norm, (UCPFunc)getkan_none};
-
-IFunc setjrec_func[] = {
-    (IFunc)setj_norm2, (IFunc)setj_norm2, (IFunc)setj_norm1, (IFunc)setj_norm1,
-    (IFunc)setj_knj, (IFunc)setj_ofs, (IFunc)setj_norm2, (IFunc)setj_atrb,
-    (IFunc)setj_norm2, (IFunc)setj_norm2, (IFunc)setj_norm1, (IFunc)setj_norm1,
-    (IFunc)setj_knj, (IFunc)setj_ofs, (IFunc)setj_norm2, (IFunc)setj_atrb};
-
-IFunc hiraknj_func[] = {
-    (IFunc)hiraknj_hira, (IFunc)hiraknj_norm, (IFunc)hiraknj_hask, (IFunc)hiraknj_kask,
-    (IFunc)hiraknj_knj, (IFunc)hiraknj_ofs, (IFunc)hiraknj_norm, (IFunc)hiraknj_atrb,
-    (IFunc)hiraknj_hira, (IFunc)hiraknj_norm, (IFunc)hiraknj_hask, (IFunc)hiraknj_kask,
-    (IFunc)hiraknj_knj, (IFunc)hiraknj_ofs, (IFunc)hiraknj_norm, (IFunc)hiraknj_atrb};
-
-UCPFunc makekan_func[] = {
-    (UCPFunc)makekan_norm, (UCPFunc)makekan_norm, (UCPFunc)makekan_1byte, (UCPFunc)makekan_1byte,
-    (UCPFunc)makekan_knj, (UCPFunc)makekan_ofs, (UCPFunc)makekan_norm, (UCPFunc)makekan_none,
-    (UCPFunc)makekan_norm, (UCPFunc)makekan_norm, (UCPFunc)makekan_1byte, (UCPFunc)makekan_1byte,
-    (UCPFunc)makekan_knj, (UCPFunc)makekan_ofs, (UCPFunc)makekan_norm, (UCPFunc)makekan_none};

lib/sj4core/getkanji.c

@@ -151,6 +151,7 @@ code2kanji(SJ4_CONTEXT u_char* s, u_char* d, u_char* ym, int yl, int flg) {
 				break;
 
 			case AIATTRIBUTE:
+			case KANAKANCOST:
 				d = getkan_none(s, d, ym, yl, flg);
 				break;
 
@@ -181,6 +182,7 @@ code2kanji(SJ4_CONTEXT u_char* s, u_char* d, u_char* ym, int yl, int flg) {
 				break;
 
 			case AIATTRIBUTE:
+			case KANAKANCOST:
 				d = getkan_none(s, d, ym, yl, FALSE);
 				break;
 

lib/sj4core/mk2claus.c

@@ -60,6 +60,16 @@ set2nd(SJ4_CONTEXT CLREC* clrec) {
 	return rec2 ? (clrec->cllen + rec2->cllen) : (clrec->cllen);
 }
 
+static int cost_penalty(CLREC* clrec) {
+	int c = clrec->jnode->cost;
+
+	if(c < 0xfff / 5 * 1) return 8;
+	if(c < 0xfff / 5 * 2) return 4;
+	if(c < 0xfff / 5 * 3) return 2;
+	if(c < 0xfff / 5 * 4) return 1;
+	return 0;
+};
+
 void mk2claus(SJ4_CONTEXT2) {
 	u_char* keepptr;
 	int	keeplen;
@@ -117,7 +127,7 @@ void mk2claus(SJ4_CONTEXT2) {
 
 		if(selcl->cl2len > clrec->cl2len) continue;
 
-		prty = priority(SJ4_CONTEXT_PASS clrec) - prty2;
+		prty = priority(SJ4_CONTEXT_PASS clrec) - prty2 + cost_penalty(clrec);
 
 		prty2++;
 

lib/sj4core/mkkouho.c

@@ -193,6 +193,7 @@ makekan(SJ4_CONTEXT u_char* s, u_char* d, int flg) {
 				break;
 
 			case AIATTRIBUTE:
+			case KANAKANCOST:
 				d = makekan_none(SJ4_CONTEXT_PASS s, d, flg);
 				break;
 
@@ -223,6 +224,7 @@ makekan(SJ4_CONTEXT u_char* s, u_char* d, int flg) {
 				break;
 
 			case AIATTRIBUTE:
+			case KANAKANCOST:
 				d = makekan_none(SJ4_CONTEXT_PASS s, d, FALSE);
 				break;
 

lib/sj4core/setjrec.c

@@ -79,6 +79,10 @@ int setj_ofs(SJ4_CONTEXT u_char* p) {
 			contf = FALSE;
 			break;
 
+		case KANAKANCOST:
+			p += 2;
+			break;
+
 		default:
 			p += setj_norm2(SJ4_CONTEXT_PASS p);
 		}
@@ -123,6 +127,10 @@ int setj_knj(SJ4_CONTEXT u_char* p) {
 			contf = FALSE;
 			break;
 
+		case KANAKANCOST:
+			p += 2;
+			break;
+
 		default:
 			p += setj_norm2(SJ4_CONTEXT_PASS p);
 		}
@@ -171,6 +179,7 @@ void setjrec(SJ4_CONTEXT u_char* tagp, int mode) {
 	int	 plen;
 	int	 nlen;
 	int	 len;
+	int	 cost = 0;
 
 	plen = getplen(tagp);
 	nlen = getnlen(tagp);
@@ -214,6 +223,11 @@ void setjrec(SJ4_CONTEXT u_char* tagp, int mode) {
 				tmp++;
 				break;
 
+			case KANAKANCOST:
+				cost = ((*tmp & 0xf) << 8) | *(tmp + 1);
+				tmp += 2;
+				break;
+
 			default:
 				tmp += setj_norm2(SJ4_CONTEXT_PASS tmp);
 			}
@@ -232,6 +246,7 @@ void setjrec(SJ4_CONTEXT u_char* tagp, int mode) {
 		rec->dicid  = curdict->dicid;
 		rec->hinsi  = gram;
 		rec->sttofs = headcode;
+		rec->cost   = cost;
 	}
 }
 
@@ -282,6 +297,10 @@ void setnumrec(SJ4_CONTEXT u_char* tagp, JREC* rec, TypeGram gram) {
 				tmp++;
 				break;
 
+			case KANAKANCOST:
+				tmp += 2;
+				break;
+
 			default:
 				tmp += setj_norm2(SJ4_CONTEXT_PASS tmp);
 			}

lib/sj4core/setkouho.c

@@ -186,6 +186,11 @@ hiraknj(SJ4_CONTEXT u_char* p) {
 			result = hiraknj_knj(SJ4_CONTEXT_PASS p, &i);
 			break;
 
+		case KANAKANCOST:
+			result = 0;
+			i      = 2;
+			break;
+
 		default:
 			result = hiraknj_hira(SJ4_CONTEXT_PASS p, &i);
 		}

src/sj4mkdic/cnvhinsi.c

@@ -62,6 +62,18 @@ int u_strcmp(u_char* a, u_char* b) {
 
 int cnvhinsi(u_char* buf) {
 	int i;
+
+	if(buf[0] == 'C') {
+		int n = atoi(buf + 1);
+
+		if(n < 0) n = 0;
+		if(n > 0xffff) n = 0xffff;
+
+		n = n * 0xfff / 0xffff;
+
+		return -(KANAKANCOST_BIT | n);
+	}
+
 #if 1
 	for(i = 0; i <= GramMax; i++) {
 		if(u_strcmp(buf, (u_char*)gramtbl[i].name) == 0) return gramtbl[i].code;

src/sj4mkdic/makelist.c

@@ -107,11 +107,18 @@ makekanji(int* yomi, int* kanji, int* atr, int* len) {
 	while(*atr) {
 		i = *atr;
 		atr++;
+
+		if(i & KANAKANCOST_BIT) {
+			i &= ~KANAKANCOST_BIT;
+			ktmp[pos++] = (KANAKANCOST | ((i >> 8) & 0xff));
+			ktmp[pos++] = (i & 0xff);
+		} else {
 #ifndef NO_ATR
 			ktmp[pos++] = (AIATTRIBUTE | ((i >> 8) & 0xff));
 			ktmp[pos++] = (i & 0xff);
 #endif
 		}
+	}
 
 	if((i = top_strcmp(yomi, kanji))) {
 		kanji += i;

src/sj4mkdic/readline.c

@@ -215,6 +215,7 @@ retry:
 
 	if(hinsi[0] == '[' && hinsi[i - 1] == ']') {
 		hinsi[i - 1] = 0;
+
 		i = cnvhinsi(hinsi + 1);
 		if(i > 0) {
 			fprintf(stderr,

src/sj4mkdic/sj4_dict_const.h

@@ -69,6 +69,7 @@
 #define AIATTRIBUTE 0xb0
 #define KANJIASSYUKU 0xc0
 #define LEADINGHANKAKU 0xd0
+#define KANAKANCOST 0xe0
 #define KANJISTREND 0x00
 #define ISASSYUKU 0x80
 #define ASSUKUUSE2BYTE 0x90
@@ -168,4 +169,6 @@
 #define HINSIREC 7
 #define DOUONREC 8
 
+#define KANAKANCOST_BIT (1 << 24)
+
 #endif

tools/wnn2sj4.py

@@ -1,6 +1,10 @@
 #!/usr/bin/env python3
 
+# THIS IS FOR SYSTEM DICTIONARY GENERATION!!!!!
+
 import sys
+import re
+from math import *
 from collections import defaultdict
 
 HINSHI = {
@@ -118,6 +122,9 @@ for lineno, line in enumerate(fp, 1):
 	if h in SKIP:
 		continue
 
+	if re.findall(r'[^あ-んー]', y):
+		continue
+
 	sj = HINSHI[h]
 
 	if sj is None:
@@ -136,8 +143,13 @@ fp.close()
 
 grouped = defaultdict(list)
 
+maxf = 0
+
 for (y, k, sj), freq in entries.items():
-	grouped[y].append((freq, k, sj))
+	if maxf < freq:
+		maxf = freq
+
+	grouped[y].append((freq, k, sj, freq))
 
 fp = open(outp, "w", encoding="utf-8")
 for y in sorted(grouped.keys()):
@@ -147,7 +159,8 @@ for y in sorted(grouped.keys()):
 
 	t = "\t" * (4 - (len(y) // 4))
 
-	for freq, k, sj in lst:
+	for freq, k, sj, freq in lst:
 		t2 = "\t" * (4 - (len(k) // 4))
-		fp.write(f"{y}{t}{k}{t2}{sj}:\n")
+		freq = floor(freq / maxf * 0xffff)
+		fp.write(f"{y}{t}{k}{t2}{sj}:[C{freq}]:\n")
 fp.close()