diff --git a/tools/mecab2sj4.py b/tools/mecab2sj4.py new file mode 100755 index 0000000..2daa4ab --- /dev/null +++ b/tools/mecab2sj4.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 + +# THIS IS FOR SYSTEM DICTIONARY GENERATION!!!!! + +import re +import sys + +FULLWIDTH_DIGITS = "0123456789" +FULLWIDTH_ALPHABET = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" +FULLWIDTH_PUNCTUATION = "!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~ " +FULLWIDTH_ALPHANUMERIC = FULLWIDTH_DIGITS + FULLWIDTH_ALPHABET +FULLWIDTH_ALL = FULLWIDTH_ALPHANUMERIC + FULLWIDTH_PUNCTUATION + +HALFWIDTH_DIGITS = "0123456789" +HALFWIDTH_ALPHABET = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" +HALFWIDTH_PUNCTUATION = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~ " +HALFWIDTH_ALPHANUMERIC = HALFWIDTH_DIGITS + HALFWIDTH_ALPHABET +HALFWIDTH_ALL = HALFWIDTH_ALPHANUMERIC + HALFWIDTH_PUNCTUATION + +KATAKANA = "アイウエオカキクケコタチツテトナニヌネノハヒフヘホマミムメモヤユヨラリルレロワヲン" +HIRAGANA = "あいうえおかきくけこたちつてとなにぬねのはひふへほまみむめもやゆよらりるれろわをん" + +if len(sys.argv) != 3: + print("usage: mecab2sj4.py input.csv output.dic", file=sys.stderr) + sys.exit(1) + +inp = sys.argv[1] +outp = sys.argv[2] + +try: + fp = open("dict/visual.plus.dic", "r") +except: + fp = open("../dict/visual.plus.dic", "r") + +dic = {} +fhtrans = str.maketrans(FULLWIDTH_ALL, HALFWIDTH_ALL) +hftrans = str.maketrans(HALFWIDTH_ALL, FULLWIDTH_ALL) +khtrans = str.maketrans(KATAKANA, HIRAGANA) + +for lineno, line in enumerate(fp, 1): + line = line.rstrip() + + if not(line): + continue + + words = list(filter(None, re.split(r'[: \t]+', line))) + for i in words[2:]: + if not(words[0] in dic): + dic[words[0]] = { + "types": [], + "write": words[1].translate(hftrans) + } + if not(i in dic[words[0]]["types"]): + dic[words[0]]["types"].append(i) + +fp = open(inp, "r") +for lineno, line in enumerate(fp, 1): + line = line.rstrip() + + if not(line): + continue + + words = line.split(',') + types = words[4:10] + read = words[11].translate(khtrans) + write = words[0].translate(fhtrans) + if not(read in dic): + next + +def reptab(s): + n = 4 - int(len(s) / 4) + + return " " * n + +fp = open(outp, "w") +s = sorted(list(dic.keys())) +for i in s: + n = 0 + print(i + reptab(i) + dic[i]["write"] + reptab(dic[i]["write"]), end="", file=fp) + for j in dic[i]["types"]: + if n == 4: + print("", file=fp) + print(i + reptab(i) + dic[i]["write"] + reptab(dic[i]["write"]), end="", file=fp) + n = 0 + print(j + ":", end="", file=fp) + n = n + 1 + print("", file=fp)