diff --git a/CMakeLists.txt b/CMakeLists.txt index 9d25ed7..33f1002 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -2,13 +2,18 @@ cmake_minimum_required(VERSION 3.11) project(sj3) +option(UTF8 "UTF8" OFF) + file(GLOB_RECURSE SJ3CORE_SRCS lib/sj3core/**.c) add_library(sj3core STATIC ${SJ3CORE_SRCS}) target_include_directories(sj3core PUBLIC include/sj3common include/sj3core) target_include_directories(sj3core PRIVATE ${CMAKE_CURRENT_BINARY_DIR}) -target_compile_definitions(sj3core PRIVATE UTF8) file(GLOB_RECURSE SJ3MKDIC_SRCS src/sj3mkdic/**.c) add_executable(sj3mkdic ${SJ3MKDIC_SRCS}) target_include_directories(sj3mkdic PRIVATE include/sj3common include/sj3core) -target_compile_definitions(sj3mkdic PRIVATE UTF8) + +if(UTF8) + target_compile_definitions(sj3core PRIVATE UTF8) + target_compile_definitions(sj3mkdic PRIVATE UTF8) +endif() diff --git a/lib/sj3core/sdepend.c b/lib/sj3core/sdepend.c index 849866c..c16ac05 100644 --- a/lib/sj3core/sdepend.c +++ b/lib/sj3core/sdepend.c @@ -256,8 +256,8 @@ opendict(char* name, char* passwd) { return dfp; } - if((fp = fopen(name, "r+")) == NULL) { - if((fp = fopen(name, "r")) == NULL) { + if((fp = fopen(name, "r+b")) == NULL) { + if((fp = fopen(name, "rb")) == NULL) { serv_errno = SJ3_CannotOpenFile; return NULL; } @@ -427,7 +427,7 @@ openstdy(char* name, char* passwd) { return NULL; } - if((fp = fopen(name, "r+")) == NULL) { + if((fp = fopen(name, "r+b")) == NULL) { serv_errno = SJ3_CannotOpenFile; goto error0; } @@ -586,7 +586,7 @@ int makedict(char* path, int idxlen, int seglen, int segnum) { return ret; } - if(!(fp = fopen(path, "w"))) { + if(!(fp = fopen(path, "wb"))) { serv_errno = SJ3_CannotCreateFile; return ret; } @@ -652,7 +652,7 @@ int makestdy(char* path, int stynum, int clstep, int cllen) { return ret; } - if(!(fp = fopen(path, "w"))) { + if(!(fp = fopen(path, "wb"))) { serv_errno = SJ3_CannotCreateFile; return ret; } diff --git a/src/sj3mkdic/char.c b/src/sj3mkdic/char.c index cc1990b..e85390b 100644 --- a/src/sj3mkdic/char.c +++ b/src/sj3mkdic/char.c @@ -39,6 +39,19 @@ #include "sj3mkdic.h" int cnvyomi(int code) { +#ifdef UTF8 + if(code == 0x30fc) return 1; + if(code == 0xff03) return 2; + if(code == 0xff20) return 3; + if('0' <= code && code <= '9') return code - '0' + 0x10; + if('A' <= code && code <= 'Z') return code - 'A' + 0x1a; + if(0x3041 <= code && code <= 0x3093) return code - 0x3041 + 0x4e; + if(code == 0x30f4) return 0xa1; + if(code == 0x30f5) return 0xa2; + if(code == 0x30f6) return 0xa3; + + return 0; +#else u_short hh; u_char high; u_char low; @@ -93,6 +106,7 @@ int cnvyomi(int code) { } return 0; +#endif } int h2kcode(int code) { @@ -193,6 +207,14 @@ void output_str(FILE* fp, char* p) { void output_int(FILE* fp, int* p) { while(*p) { +#ifdef UTF8 + char buf[5]; + + utf8_print(buf, *p); + fputs(buf, fp); + + p++; +#else if(*p < 0x100) { fputc(*p, fp); p++; @@ -212,10 +234,40 @@ void output_int(FILE* fp, int* p) { fputc(*p & 0xff, fp); p++; } +#endif } } int yomi2zen(int code) { +#ifdef UTF8 + switch(code) { + case 1: + return 0x30fc; + case 2: + return 0xff03; + case 3: + return 0xff20; + } + + if(0x10 <= code && code <= 0x19) return code - 0x10 + '0'; + if(0x1a <= code && code <= 0x4d) return code - 0x1a + 'A'; + if(0x4e <= code && code <= 0xa0) return code - 0x4e + 0x3041; + if(code == 0xa1) return 0x30f4; + if(code == 0xa2) return 0x30f5; + if(code == 0xa3) return 0x30f6; + + if(code == 0x30fc) return 1; + if(code == 0xff03) return 2; + if(code == 0xff20) return 3; + if('0' <= code && code <= '9') return code - '0' + 0x10; + if('A' <= code && code <= 'Z') return code - 'A' + 0x1a; + if(0x3041 <= code && code <= 0x3093) return code - 0x3041 + 0x4e; + if(code == 0x30f4) return 0xa1; + if(code == 0x30f5) return 0xa2; + if(code == 0x30f6) return 0xa3; + + return code; +#else static char num[] = { '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F'}; @@ -255,14 +307,23 @@ int yomi2zen(int code) { return (0xa400 + code - 0x4e + 0xa1); return ((num[(code >> 4) & 0x0f] << 8) | num[code & 0x0f]); +#endif } void output_yomi(FILE* fp, u_char* p) { - int i; + int i; + char buf[5]; while(*p) { i = yomi2zen(*p++); + +#ifdef UTF8 + utf8_print(buf, i); + + fputs(buf, fp); +#else fputc((i >> 8) & 0xff, fp); fputc(i & 0xff, fp); +#endif } } diff --git a/src/sj3mkdic/file.c b/src/sj3mkdic/file.c index dbc66b8..8bfcc6e 100644 --- a/src/sj3mkdic/file.c +++ b/src/sj3mkdic/file.c @@ -118,6 +118,8 @@ size_t Fread(char* p, int s, int n, FILE* fp) { if(fread(p, s, n, fp) == n) return n; + printf("!\n"); + fprintf(stderr, "Read error in %s\n", get_fname(fp)); exit(1); } diff --git a/src/sj3mkdic/makedict.c b/src/sj3mkdic/makedict.c index ee6d308..db1dfc9 100644 --- a/src/sj3mkdic/makedict.c +++ b/src/sj3mkdic/makedict.c @@ -117,7 +117,7 @@ void parse(int argc, char** argv) { p = argv[1]; idxtop = (*p == '-') ? get_idxlist(p + 1) : make_idxlist(p); - outfp = Fopen(argv[2], "w"); + outfp = Fopen(argv[2], "wb"); } /* XXX: strtonum??? */ diff --git a/src/sj3mkdic/makelist.c b/src/sj3mkdic/makelist.c index 7b46bbe..9931b6e 100644 --- a/src/sj3mkdic/makelist.c +++ b/src/sj3mkdic/makelist.c @@ -172,7 +172,13 @@ makeyomi(int* yomi) { for(i = 0; *y;) { j = cnvyomi(*y++); if(j == 0) { - fprintf(stderr, "\311\324\300\265\244\312\312\270\273\372\244\254\306\311\244\337\244\313\273\310\244\357\244\354\244\306\244\244\244\336\244\271\n"); + fprintf(stderr, +#ifdef UTF8 + "不正な文字が読みに使われています\n" +#else + "\311\324\300\265\244\312\312\270\273\372\244\254\306\311\244\337\244\313\273\310\244\357\244\354\244\306\244\244\244\336\244\271\n" +#endif + ); output_int(stderr, yomi); fputc('\n', stderr); exit(1); diff --git a/src/sj3mkdic/makeseg.c b/src/sj3mkdic/makeseg.c index 1d3b32e..ac79be1 100644 --- a/src/sj3mkdic/makeseg.c +++ b/src/sj3mkdic/makeseg.c @@ -60,7 +60,13 @@ set_ofsask(u_char* src, u_char* dst) { break; case LEADINGHANKAKU: #ifndef USEHANKAKUINDICT - fprintf(stderr, "\302\260\300\255\245\263\241\274\245\311\241\246\245\250\245\351\241\274\n"); + fprintf(stderr, +#ifdef UTF8 + "\264\301\273\372\260\265\275\314\312\270\273\372\316\363\244\254\304\271\244\271\244\256\244\353\n" +#else + "\302\260\300\255\245\263\241\274\245\311\241\246\245\250\245\351\241\274\n" +#endif + ); exit(1); #else *dst++ = *ptr++; @@ -69,7 +75,13 @@ set_ofsask(u_char* src, u_char* dst) { break; #endif case OFFSETASSYUKU: - fprintf(stderr, "\306\363\275\305\244\313\245\252\245\325\245\273\245\303\245\310\244\254\273\262\276\310\244\265\244\354\244\306\244\244\244\353\n"); + fprintf(stderr, +#ifdef UTF8 + "二重にオフセットが参照されている\n" +#else + "\306\363\275\305\244\313\245\252\245\325\245\273\245\303\245\310\244\254\273\262\276\310\244\265\244\354\244\306\244\244\244\353\n" +#endif + ); exit(1); case AIATTRIBUTE: *dst++ = *ptr++; @@ -103,7 +115,13 @@ int make_knjstr(u_char* src, int len, u_char* dst) { break; case LEADINGHANKAKU: #ifndef USEHANKAKUINDICT - fprintf(stderr, "\302\260\300\255\245\263\241\274\245\311\241\246\245\250\245\351\241\274\n"); + fprintf(stderr, +#ifdef UTF8 + "\264\301\273\372\260\265\275\314\312\270\273\372\316\363\244\254\304\271\244\271\244\256\244\353\n" +#else + "\302\260\300\255\245\263\241\274\245\311\241\246\245\250\245\351\241\274\n" +#endif + ); exit(1); #else *dst++ = *src++; @@ -149,7 +167,13 @@ make_knjask() { p = (u_char*)Malloc(len); if(!p) { - fprintf(stderr, "\245\341\245\342\245\352\244\254\302\255\244\352\244\336\244\273\244\363\n"); + fprintf(stderr, +#ifdef UTF8 + "メモリが足りません\n" +#else + "\245\341\245\342\245\352\244\254\302\255\244\352\244\336\244\273\244\363\n" +#endif + ); exit(1); } @@ -192,7 +216,13 @@ void makeseg() { i = make_knjask(); if(i >= 0x100) { - fprintf(stderr, "\264\301\273\372\260\265\275\314\312\270\273\372\316\363\244\254\304\271\244\271\244\256\244\353\n"); + fprintf(stderr, +#ifdef UTF8 + "漢字圧縮文字列が長すぎる\n" +#else + "\264\301\273\372\260\265\275\314\312\270\273\372\316\363\244\254\304\271\244\271\244\256\244\353\n" +#endif + ); exit(1); } @@ -270,7 +300,13 @@ void makeseg() { if(idxpos < MAININDEXLENGTH) { mindex[idxpos++] = *p; } else { - fprintf(stderr, "\245\244\245\363\245\307\245\303\245\257\245\271\244\254\244\242\244\325\244\354\244\336\244\267\244\277\n"); + fprintf(stderr, +#ifdef UTF8 + "インデックスがあふれました\n" +#else + "\245\244\245\363\245\307\245\303\245\257\245\271\244\254\244\242\244\325\244\354\244\336\244\267\244\277\n" +#endif + ); exit(1); } if(!*p++) break; @@ -278,7 +314,13 @@ void makeseg() { if(idxpos < MAININDEXLENGTH) { mindex[idxpos] = 0; } else { - fprintf(stderr, "\245\244\245\363\245\307\245\303\245\257\245\271\244\254\244\242\244\325\244\354\244\336\244\267\244\277\n"); + fprintf(stderr, +#ifdef UTF8 + "インデックスがあふれました\n" +#else + "\245\244\245\363\245\307\245\303\245\257\245\271\244\254\244\242\244\325\244\354\244\336\244\267\244\277\n" +#endif + ); exit(1); } @@ -288,20 +330,47 @@ void makeseg() { if(Fseek(outfp, (long)(HEADERLENGTH + COMMENTLENGTH + MAININDEXLENGTH + idxnum * MAINSEGMENTLENGTH), 0) < 0) { - fprintf(stderr, "\275\320\316\317\245\325\245\241\245\244\245\353\244\307\245\267\241\274\245\257\245\250\245\351\241\274\n"); + fprintf(stderr, +#ifdef UTF8 + "出力ファイルでシークエラー\n" +#else + "\275\320\316\317\245\325\245\241\245\244\245\353\244\307\245\267\241\274\245\257\245\250\245\351\241\274\n" +#endif + ); exit(1); } if(Fwrite((char*)buf, sizeof(buf), 1, outfp) != 1) { - fprintf(stderr, "\275\320\316\317\245\325\245\241\245\244\245\353\244\307\245\351\245\244\245\310\245\250\245\351\241\274\n"); + fprintf(stderr, +#ifdef UTF8 + "出力ファイルでライトエラー\n" +#else + "\275\320\316\317\245\325\245\241\245\244\245\353\244\307\245\351\245\244\245\310\245\250\245\351\241\274\n" +#endif + ); exit(1); } Fflush(outfp); - printf("\245\273\245\260\245\341\245\363\245\310\310\326\271\346:%d:", idxnum++); + printf( +#ifdef UTF8 + "セグメント番号:%d:" +#else + "\245\273\245\260\245\341\245\363\245\310\310\326\271\346:%d:" +#endif + , + idxnum++); + output_yomi(stdout, douon_ptr->yptr); + +#ifdef UTF8 + printf("\tセグメント長:%d", i); + printf("\t同音語数:%d\n", douon_num); + printf("漢字圧縮文字列:"); +#else printf("\t\245\273\245\260\245\341\245\363\245\310\304\271:%d", i); printf("\t\306\261\262\273\270\354\277\364:%d\n", douon_num); printf("\264\301\273\372\260\265\275\314\312\270\273\372\316\363:"); +#endif for(i = j = 0; i < askknj_num; i++) { putchar('\t'); if(i && j == 0) putchar('\t'); @@ -319,8 +388,15 @@ void makeseg() { return; err: - fprintf(stderr, "\245\273\245\260\245\341\245\363\245\310\271\275\300\256\303\346\244\313\245\320\245\303\245\325\245\241\244\254\244\242\244\325\244\354\244\277\n"); - fprintf(stderr, "\245\273\245\260\245\341\245\363\245\310\300\350\306\254\244\316\306\311\244\337:"); + fprintf(stderr, +#ifdef UTF8 + "セグメント構成中にバッファがあふれた\n" + "セグメント先頭の読み:" +#else + "\245\273\245\260\245\341\245\363\245\310\271\275\300\256\303\346\244\313\245\320\245\303\245\325\245\241\244\254\244\242\244\325\244\354\244\277\n" + "\245\273\245\260\245\341\245\363\245\310\300\350\306\254\244\316\306\311\244\337:" +#endif + ); output_yomi(stderr, douon_ptr->yptr); fputc('\n', stderr); diff --git a/src/sj3mkdic/readline.c b/src/sj3mkdic/readline.c index 175374d..c5ec46b 100644 --- a/src/sj3mkdic/readline.c +++ b/src/sj3mkdic/readline.c @@ -79,7 +79,7 @@ readchar() { if(i == 1) { n = c[0]; } else { - n |= c[0] & (0xff >> (8 - i)); + n |= c[0] & (0xff >> (8 - i - 1)); for(j = 1; j < i; j++) { n = n << 6; @@ -155,7 +155,6 @@ retry: error(ILLEGALFORMAT); #ifdef UTF8 - printf("%x\n", c); if(c >= 0x10000) { hinsi[i++] = ((c >> (6 * 3)) & 7) | 0xf0; diff --git a/src/sj3mkdic/sj3mkdic.h b/src/sj3mkdic/sj3mkdic.h index dff5ba0..18d0ea5 100644 --- a/src/sj3mkdic/sj3mkdic.h +++ b/src/sj3mkdic/sj3mkdic.h @@ -116,12 +116,13 @@ int* readline(); void setline(void (*)(int*, int*, int, int*)); /* string.c */ -int bubun_str(u_char*, int, u_char*, int); -int overlap_str(u_char*, int, u_char*, int); -int istrlen(int*); -int istrcmp(int*, int*); -int top_strcmp(int*, int*); -int last_strcmp(int*, int*); -int string_cmp(u_char*, int, u_char*, int); +int bubun_str(u_char*, int, u_char*, int); +int overlap_str(u_char*, int, u_char*, int); +int istrlen(int*); +int istrcmp(int*, int*); +int top_strcmp(int*, int*); +int last_strcmp(int*, int*); +int string_cmp(u_char*, int, u_char*, int); +void utf8_print(char*, int); #endif /* SJ3MKDIC_H */ diff --git a/src/sj3mkdic/string.c b/src/sj3mkdic/string.c index 36226ca..e6aad8b 100644 --- a/src/sj3mkdic/string.c +++ b/src/sj3mkdic/string.c @@ -173,3 +173,29 @@ int string_cmp(u_char* p1, int l1, u_char* p2, int l2) { if(l1 > l2) return 1; return -1; } + +void utf8_print(char* out, int c) { + int i = 0, j, k; + + if(c >= 0x10000) { + out[i++] = ((c >> (6 * 3)) & 7) | 0xf0; + + j = 3; + } else if(c >= 0x800) { + out[i++] = ((c >> (6 * 2)) & 15) | 0xe0; + + j = 2; + } else if(c >= 0x80) { + out[i++] = ((c >> (6 * 1)) & 31) | 0xc0; + + j = 1; + } else { + out[i++] = c; + } + + for(k = 0; k < j; k++) { + out[i++] = ((c >> (6 * (j - k - 1))) & 63) | 0x80; + } + + out[i++] = 0; +}