UTF8 as option
This commit is contained in:
parent
f0700d97a6
commit
77dc93e072
10 changed files with 207 additions and 31 deletions
|
|
@ -2,13 +2,18 @@ cmake_minimum_required(VERSION 3.11)
|
|||
|
||||
project(sj3)
|
||||
|
||||
option(UTF8 "UTF8" OFF)
|
||||
|
||||
file(GLOB_RECURSE SJ3CORE_SRCS lib/sj3core/**.c)
|
||||
add_library(sj3core STATIC ${SJ3CORE_SRCS})
|
||||
target_include_directories(sj3core PUBLIC include/sj3common include/sj3core)
|
||||
target_include_directories(sj3core PRIVATE ${CMAKE_CURRENT_BINARY_DIR})
|
||||
target_compile_definitions(sj3core PRIVATE UTF8)
|
||||
|
||||
file(GLOB_RECURSE SJ3MKDIC_SRCS src/sj3mkdic/**.c)
|
||||
add_executable(sj3mkdic ${SJ3MKDIC_SRCS})
|
||||
target_include_directories(sj3mkdic PRIVATE include/sj3common include/sj3core)
|
||||
target_compile_definitions(sj3mkdic PRIVATE UTF8)
|
||||
|
||||
if(UTF8)
|
||||
target_compile_definitions(sj3core PRIVATE UTF8)
|
||||
target_compile_definitions(sj3mkdic PRIVATE UTF8)
|
||||
endif()
|
||||
|
|
|
|||
|
|
@ -256,8 +256,8 @@ opendict(char* name, char* passwd) {
|
|||
return dfp;
|
||||
}
|
||||
|
||||
if((fp = fopen(name, "r+")) == NULL) {
|
||||
if((fp = fopen(name, "r")) == NULL) {
|
||||
if((fp = fopen(name, "r+b")) == NULL) {
|
||||
if((fp = fopen(name, "rb")) == NULL) {
|
||||
serv_errno = SJ3_CannotOpenFile;
|
||||
return NULL;
|
||||
}
|
||||
|
|
@ -427,7 +427,7 @@ openstdy(char* name, char* passwd) {
|
|||
return NULL;
|
||||
}
|
||||
|
||||
if((fp = fopen(name, "r+")) == NULL) {
|
||||
if((fp = fopen(name, "r+b")) == NULL) {
|
||||
serv_errno = SJ3_CannotOpenFile;
|
||||
goto error0;
|
||||
}
|
||||
|
|
@ -586,7 +586,7 @@ int makedict(char* path, int idxlen, int seglen, int segnum) {
|
|||
return ret;
|
||||
}
|
||||
|
||||
if(!(fp = fopen(path, "w"))) {
|
||||
if(!(fp = fopen(path, "wb"))) {
|
||||
serv_errno = SJ3_CannotCreateFile;
|
||||
return ret;
|
||||
}
|
||||
|
|
@ -652,7 +652,7 @@ int makestdy(char* path, int stynum, int clstep, int cllen) {
|
|||
return ret;
|
||||
}
|
||||
|
||||
if(!(fp = fopen(path, "w"))) {
|
||||
if(!(fp = fopen(path, "wb"))) {
|
||||
serv_errno = SJ3_CannotCreateFile;
|
||||
return ret;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -39,6 +39,19 @@
|
|||
#include "sj3mkdic.h"
|
||||
|
||||
int cnvyomi(int code) {
|
||||
#ifdef UTF8
|
||||
if(code == 0x30fc) return 1;
|
||||
if(code == 0xff03) return 2;
|
||||
if(code == 0xff20) return 3;
|
||||
if('0' <= code && code <= '9') return code - '0' + 0x10;
|
||||
if('A' <= code && code <= 'Z') return code - 'A' + 0x1a;
|
||||
if(0x3041 <= code && code <= 0x3093) return code - 0x3041 + 0x4e;
|
||||
if(code == 0x30f4) return 0xa1;
|
||||
if(code == 0x30f5) return 0xa2;
|
||||
if(code == 0x30f6) return 0xa3;
|
||||
|
||||
return 0;
|
||||
#else
|
||||
u_short hh;
|
||||
u_char high;
|
||||
u_char low;
|
||||
|
|
@ -93,6 +106,7 @@ int cnvyomi(int code) {
|
|||
}
|
||||
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
int h2kcode(int code) {
|
||||
|
|
@ -193,6 +207,14 @@ void output_str(FILE* fp, char* p) {
|
|||
|
||||
void output_int(FILE* fp, int* p) {
|
||||
while(*p) {
|
||||
#ifdef UTF8
|
||||
char buf[5];
|
||||
|
||||
utf8_print(buf, *p);
|
||||
fputs(buf, fp);
|
||||
|
||||
p++;
|
||||
#else
|
||||
if(*p < 0x100) {
|
||||
fputc(*p, fp);
|
||||
p++;
|
||||
|
|
@ -212,10 +234,40 @@ void output_int(FILE* fp, int* p) {
|
|||
fputc(*p & 0xff, fp);
|
||||
p++;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
int yomi2zen(int code) {
|
||||
#ifdef UTF8
|
||||
switch(code) {
|
||||
case 1:
|
||||
return 0x30fc;
|
||||
case 2:
|
||||
return 0xff03;
|
||||
case 3:
|
||||
return 0xff20;
|
||||
}
|
||||
|
||||
if(0x10 <= code && code <= 0x19) return code - 0x10 + '0';
|
||||
if(0x1a <= code && code <= 0x4d) return code - 0x1a + 'A';
|
||||
if(0x4e <= code && code <= 0xa0) return code - 0x4e + 0x3041;
|
||||
if(code == 0xa1) return 0x30f4;
|
||||
if(code == 0xa2) return 0x30f5;
|
||||
if(code == 0xa3) return 0x30f6;
|
||||
|
||||
if(code == 0x30fc) return 1;
|
||||
if(code == 0xff03) return 2;
|
||||
if(code == 0xff20) return 3;
|
||||
if('0' <= code && code <= '9') return code - '0' + 0x10;
|
||||
if('A' <= code && code <= 'Z') return code - 'A' + 0x1a;
|
||||
if(0x3041 <= code && code <= 0x3093) return code - 0x3041 + 0x4e;
|
||||
if(code == 0x30f4) return 0xa1;
|
||||
if(code == 0x30f5) return 0xa2;
|
||||
if(code == 0x30f6) return 0xa3;
|
||||
|
||||
return code;
|
||||
#else
|
||||
static char num[] = {
|
||||
'0', '1', '2', '3', '4', '5', '6', '7',
|
||||
'8', '9', 'A', 'B', 'C', 'D', 'E', 'F'};
|
||||
|
|
@ -255,14 +307,23 @@ int yomi2zen(int code) {
|
|||
return (0xa400 + code - 0x4e + 0xa1);
|
||||
|
||||
return ((num[(code >> 4) & 0x0f] << 8) | num[code & 0x0f]);
|
||||
#endif
|
||||
}
|
||||
|
||||
void output_yomi(FILE* fp, u_char* p) {
|
||||
int i;
|
||||
int i;
|
||||
char buf[5];
|
||||
|
||||
while(*p) {
|
||||
i = yomi2zen(*p++);
|
||||
|
||||
#ifdef UTF8
|
||||
utf8_print(buf, i);
|
||||
|
||||
fputs(buf, fp);
|
||||
#else
|
||||
fputc((i >> 8) & 0xff, fp);
|
||||
fputc(i & 0xff, fp);
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -118,6 +118,8 @@ size_t
|
|||
Fread(char* p, int s, int n, FILE* fp) {
|
||||
if(fread(p, s, n, fp) == n) return n;
|
||||
|
||||
printf("!\n");
|
||||
|
||||
fprintf(stderr, "Read error in %s\n", get_fname(fp));
|
||||
exit(1);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -117,7 +117,7 @@ void parse(int argc, char** argv) {
|
|||
p = argv[1];
|
||||
idxtop = (*p == '-') ? get_idxlist(p + 1) : make_idxlist(p);
|
||||
|
||||
outfp = Fopen(argv[2], "w");
|
||||
outfp = Fopen(argv[2], "wb");
|
||||
}
|
||||
|
||||
/* XXX: strtonum??? */
|
||||
|
|
|
|||
|
|
@ -172,7 +172,13 @@ makeyomi(int* yomi) {
|
|||
for(i = 0; *y;) {
|
||||
j = cnvyomi(*y++);
|
||||
if(j == 0) {
|
||||
fprintf(stderr, "\311\324\300\265\244\312\312\270\273\372\244\254\306\311\244\337\244\313\273\310\244\357\244\354\244\306\244\244\244\336\244\271\n");
|
||||
fprintf(stderr,
|
||||
#ifdef UTF8
|
||||
"不正な文字が読みに使われています\n"
|
||||
#else
|
||||
"\311\324\300\265\244\312\312\270\273\372\244\254\306\311\244\337\244\313\273\310\244\357\244\354\244\306\244\244\244\336\244\271\n"
|
||||
#endif
|
||||
);
|
||||
output_int(stderr, yomi);
|
||||
fputc('\n', stderr);
|
||||
exit(1);
|
||||
|
|
|
|||
|
|
@ -60,7 +60,13 @@ set_ofsask(u_char* src, u_char* dst) {
|
|||
break;
|
||||
case LEADINGHANKAKU:
|
||||
#ifndef USEHANKAKUINDICT
|
||||
fprintf(stderr, "\302\260\300\255\245\263\241\274\245\311\241\246\245\250\245\351\241\274\n");
|
||||
fprintf(stderr,
|
||||
#ifdef UTF8
|
||||
"\264\301\273\372\260\265\275\314\312\270\273\372\316\363\244\254\304\271\244\271\244\256\244\353\n"
|
||||
#else
|
||||
"\302\260\300\255\245\263\241\274\245\311\241\246\245\250\245\351\241\274\n"
|
||||
#endif
|
||||
);
|
||||
exit(1);
|
||||
#else
|
||||
*dst++ = *ptr++;
|
||||
|
|
@ -69,7 +75,13 @@ set_ofsask(u_char* src, u_char* dst) {
|
|||
break;
|
||||
#endif
|
||||
case OFFSETASSYUKU:
|
||||
fprintf(stderr, "\306\363\275\305\244\313\245\252\245\325\245\273\245\303\245\310\244\254\273\262\276\310\244\265\244\354\244\306\244\244\244\353\n");
|
||||
fprintf(stderr,
|
||||
#ifdef UTF8
|
||||
"二重にオフセットが参照されている\n"
|
||||
#else
|
||||
"\306\363\275\305\244\313\245\252\245\325\245\273\245\303\245\310\244\254\273\262\276\310\244\265\244\354\244\306\244\244\244\353\n"
|
||||
#endif
|
||||
);
|
||||
exit(1);
|
||||
case AIATTRIBUTE:
|
||||
*dst++ = *ptr++;
|
||||
|
|
@ -103,7 +115,13 @@ int make_knjstr(u_char* src, int len, u_char* dst) {
|
|||
break;
|
||||
case LEADINGHANKAKU:
|
||||
#ifndef USEHANKAKUINDICT
|
||||
fprintf(stderr, "\302\260\300\255\245\263\241\274\245\311\241\246\245\250\245\351\241\274\n");
|
||||
fprintf(stderr,
|
||||
#ifdef UTF8
|
||||
"\264\301\273\372\260\265\275\314\312\270\273\372\316\363\244\254\304\271\244\271\244\256\244\353\n"
|
||||
#else
|
||||
"\302\260\300\255\245\263\241\274\245\311\241\246\245\250\245\351\241\274\n"
|
||||
#endif
|
||||
);
|
||||
exit(1);
|
||||
#else
|
||||
*dst++ = *src++;
|
||||
|
|
@ -149,7 +167,13 @@ make_knjask() {
|
|||
|
||||
p = (u_char*)Malloc(len);
|
||||
if(!p) {
|
||||
fprintf(stderr, "\245\341\245\342\245\352\244\254\302\255\244\352\244\336\244\273\244\363\n");
|
||||
fprintf(stderr,
|
||||
#ifdef UTF8
|
||||
"メモリが足りません\n"
|
||||
#else
|
||||
"\245\341\245\342\245\352\244\254\302\255\244\352\244\336\244\273\244\363\n"
|
||||
#endif
|
||||
);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
|
|
@ -192,7 +216,13 @@ void makeseg() {
|
|||
|
||||
i = make_knjask();
|
||||
if(i >= 0x100) {
|
||||
fprintf(stderr, "\264\301\273\372\260\265\275\314\312\270\273\372\316\363\244\254\304\271\244\271\244\256\244\353\n");
|
||||
fprintf(stderr,
|
||||
#ifdef UTF8
|
||||
"漢字圧縮文字列が長すぎる\n"
|
||||
#else
|
||||
"\264\301\273\372\260\265\275\314\312\270\273\372\316\363\244\254\304\271\244\271\244\256\244\353\n"
|
||||
#endif
|
||||
);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
|
|
@ -270,7 +300,13 @@ void makeseg() {
|
|||
if(idxpos < MAININDEXLENGTH) {
|
||||
mindex[idxpos++] = *p;
|
||||
} else {
|
||||
fprintf(stderr, "\245\244\245\363\245\307\245\303\245\257\245\271\244\254\244\242\244\325\244\354\244\336\244\267\244\277\n");
|
||||
fprintf(stderr,
|
||||
#ifdef UTF8
|
||||
"インデックスがあふれました\n"
|
||||
#else
|
||||
"\245\244\245\363\245\307\245\303\245\257\245\271\244\254\244\242\244\325\244\354\244\336\244\267\244\277\n"
|
||||
#endif
|
||||
);
|
||||
exit(1);
|
||||
}
|
||||
if(!*p++) break;
|
||||
|
|
@ -278,7 +314,13 @@ void makeseg() {
|
|||
if(idxpos < MAININDEXLENGTH) {
|
||||
mindex[idxpos] = 0;
|
||||
} else {
|
||||
fprintf(stderr, "\245\244\245\363\245\307\245\303\245\257\245\271\244\254\244\242\244\325\244\354\244\336\244\267\244\277\n");
|
||||
fprintf(stderr,
|
||||
#ifdef UTF8
|
||||
"インデックスがあふれました\n"
|
||||
#else
|
||||
"\245\244\245\363\245\307\245\303\245\257\245\271\244\254\244\242\244\325\244\354\244\336\244\267\244\277\n"
|
||||
#endif
|
||||
);
|
||||
exit(1);
|
||||
}
|
||||
|
||||
|
|
@ -288,20 +330,47 @@ void makeseg() {
|
|||
if(Fseek(outfp,
|
||||
(long)(HEADERLENGTH + COMMENTLENGTH + MAININDEXLENGTH + idxnum * MAINSEGMENTLENGTH),
|
||||
0) < 0) {
|
||||
fprintf(stderr, "\275\320\316\317\245\325\245\241\245\244\245\353\244\307\245\267\241\274\245\257\245\250\245\351\241\274\n");
|
||||
fprintf(stderr,
|
||||
#ifdef UTF8
|
||||
"出力ファイルでシークエラー\n"
|
||||
#else
|
||||
"\275\320\316\317\245\325\245\241\245\244\245\353\244\307\245\267\241\274\245\257\245\250\245\351\241\274\n"
|
||||
#endif
|
||||
);
|
||||
exit(1);
|
||||
}
|
||||
if(Fwrite((char*)buf, sizeof(buf), 1, outfp) != 1) {
|
||||
fprintf(stderr, "\275\320\316\317\245\325\245\241\245\244\245\353\244\307\245\351\245\244\245\310\245\250\245\351\241\274\n");
|
||||
fprintf(stderr,
|
||||
#ifdef UTF8
|
||||
"出力ファイルでライトエラー\n"
|
||||
#else
|
||||
"\275\320\316\317\245\325\245\241\245\244\245\353\244\307\245\351\245\244\245\310\245\250\245\351\241\274\n"
|
||||
#endif
|
||||
);
|
||||
exit(1);
|
||||
}
|
||||
Fflush(outfp);
|
||||
|
||||
printf("\245\273\245\260\245\341\245\363\245\310\310\326\271\346:%d:", idxnum++);
|
||||
printf(
|
||||
#ifdef UTF8
|
||||
"セグメント番号:%d:"
|
||||
#else
|
||||
"\245\273\245\260\245\341\245\363\245\310\310\326\271\346:%d:"
|
||||
#endif
|
||||
,
|
||||
idxnum++);
|
||||
|
||||
output_yomi(stdout, douon_ptr->yptr);
|
||||
|
||||
#ifdef UTF8
|
||||
printf("\tセグメント長:%d", i);
|
||||
printf("\t同音語数:%d\n", douon_num);
|
||||
printf("漢字圧縮文字列:");
|
||||
#else
|
||||
printf("\t\245\273\245\260\245\341\245\363\245\310\304\271:%d", i);
|
||||
printf("\t\306\261\262\273\270\354\277\364:%d\n", douon_num);
|
||||
printf("\264\301\273\372\260\265\275\314\312\270\273\372\316\363:");
|
||||
#endif
|
||||
for(i = j = 0; i < askknj_num; i++) {
|
||||
putchar('\t');
|
||||
if(i && j == 0) putchar('\t');
|
||||
|
|
@ -319,8 +388,15 @@ void makeseg() {
|
|||
return;
|
||||
|
||||
err:
|
||||
fprintf(stderr, "\245\273\245\260\245\341\245\363\245\310\271\275\300\256\303\346\244\313\245\320\245\303\245\325\245\241\244\254\244\242\244\325\244\354\244\277\n");
|
||||
fprintf(stderr, "\245\273\245\260\245\341\245\363\245\310\300\350\306\254\244\316\306\311\244\337:");
|
||||
fprintf(stderr,
|
||||
#ifdef UTF8
|
||||
"セグメント構成中にバッファがあふれた\n"
|
||||
"セグメント先頭の読み:"
|
||||
#else
|
||||
"\245\273\245\260\245\341\245\363\245\310\271\275\300\256\303\346\244\313\245\320\245\303\245\325\245\241\244\254\244\242\244\325\244\354\244\277\n"
|
||||
"\245\273\245\260\245\341\245\363\245\310\300\350\306\254\244\316\306\311\244\337:"
|
||||
#endif
|
||||
);
|
||||
output_yomi(stderr, douon_ptr->yptr);
|
||||
fputc('\n', stderr);
|
||||
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ readchar() {
|
|||
if(i == 1) {
|
||||
n = c[0];
|
||||
} else {
|
||||
n |= c[0] & (0xff >> (8 - i));
|
||||
n |= c[0] & (0xff >> (8 - i - 1));
|
||||
|
||||
for(j = 1; j < i; j++) {
|
||||
n = n << 6;
|
||||
|
|
@ -155,7 +155,6 @@ retry:
|
|||
error(ILLEGALFORMAT);
|
||||
|
||||
#ifdef UTF8
|
||||
printf("%x\n", c);
|
||||
if(c >= 0x10000) {
|
||||
hinsi[i++] = ((c >> (6 * 3)) & 7) | 0xf0;
|
||||
|
||||
|
|
|
|||
|
|
@ -116,12 +116,13 @@ int* readline();
|
|||
void setline(void (*)(int*, int*, int, int*));
|
||||
|
||||
/* string.c */
|
||||
int bubun_str(u_char*, int, u_char*, int);
|
||||
int overlap_str(u_char*, int, u_char*, int);
|
||||
int istrlen(int*);
|
||||
int istrcmp(int*, int*);
|
||||
int top_strcmp(int*, int*);
|
||||
int last_strcmp(int*, int*);
|
||||
int string_cmp(u_char*, int, u_char*, int);
|
||||
int bubun_str(u_char*, int, u_char*, int);
|
||||
int overlap_str(u_char*, int, u_char*, int);
|
||||
int istrlen(int*);
|
||||
int istrcmp(int*, int*);
|
||||
int top_strcmp(int*, int*);
|
||||
int last_strcmp(int*, int*);
|
||||
int string_cmp(u_char*, int, u_char*, int);
|
||||
void utf8_print(char*, int);
|
||||
|
||||
#endif /* SJ3MKDIC_H */
|
||||
|
|
|
|||
|
|
@ -173,3 +173,29 @@ int string_cmp(u_char* p1, int l1, u_char* p2, int l2) {
|
|||
if(l1 > l2) return 1;
|
||||
return -1;
|
||||
}
|
||||
|
||||
void utf8_print(char* out, int c) {
|
||||
int i = 0, j, k;
|
||||
|
||||
if(c >= 0x10000) {
|
||||
out[i++] = ((c >> (6 * 3)) & 7) | 0xf0;
|
||||
|
||||
j = 3;
|
||||
} else if(c >= 0x800) {
|
||||
out[i++] = ((c >> (6 * 2)) & 15) | 0xe0;
|
||||
|
||||
j = 2;
|
||||
} else if(c >= 0x80) {
|
||||
out[i++] = ((c >> (6 * 1)) & 31) | 0xc0;
|
||||
|
||||
j = 1;
|
||||
} else {
|
||||
out[i++] = c;
|
||||
}
|
||||
|
||||
for(k = 0; k < j; k++) {
|
||||
out[i++] = ((c >> (6 * (j - k - 1))) & 63) | 0x80;
|
||||
}
|
||||
|
||||
out[i++] = 0;
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue