Skip to content

Commit

Permalink
Add merged charset
Browse files Browse the repository at this point in the history
  • Loading branch information
drojf committed Feb 3, 2024
1 parent 3585729 commit 3e7f7aa
Show file tree
Hide file tree
Showing 2 changed files with 50 additions and 0 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
from pathlib import Path
import re

def load_existing_list(path):
with open(path, encoding='utf-8', newline='') as f:
return f.read()

japanese_list = '../msgothic_0_charset_Japanese.txt'
multilang_list = '../msgothic_2_charset_OtherLang.txt'
out_char_list = '../msgothic_2_charset_JP_and_OtherLang.txt'

jp = load_existing_list(japanese_list)
multi = load_existing_list(multilang_list)

chars_to_add = set(jp)
existing_chars = set(multi)


with open(out_char_list, 'w', encoding='utf-8', newline='') as f:
for i, c in enumerate(multi):
f.write(c)

# This is very bad for performance if there are lots of new chars found, but it works for now to maintain ordering
remove_list = []
for new_character in chars_to_add:
if new_character < c:
f.write(new_character)
remove_list.append(new_character)
print(f"Inserting new character {new_character} at position {i} as it is less than {c}")

for item in remove_list:
chars_to_add.remove(item)

remove_list = []
for char in chars_to_add:
if char not in existing_chars:
f.write(char)
else:
print(f"WARNING: character {char} already exists, skipping")
remove_list.append(char)

for item in remove_list:
chars_to_add.remove(item)


if chars_to_add:
raise Exception(f"One or more characters were not added {chars_to_add}")
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@


! "!#"$#%$&%'&(')(*)+*,+-,.-/.0/102132435465768798:9;:<;=<>=?>@?A@BACBDCEDFEGFHGIHJIKJLKMLNMONPOQPRQSRTSUTVUWVXWYXZY[Z\[]\^]_^`_a`bacbdcedfegfhgihjikjlkmlnmonpoqprqsrtsutvuwvxwyxzy{z|{}|~} ~¡¢£¤¥¦§¨©ª«¬«­®¯°±²³´µ¶·¸¹º»¼½¾¿ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×Ø×ÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿĀāĂ㥹ĆćĈĉĊċČčĎďĐđĒēĔĕĖėĘęĚěĜĝĞğĠġĢģĤĥĦħĨĩĪīĬĭĮįİıIJijĴĵĶķĸĹĺĻļĽľĿŀŁłŃńŅņŇňʼnŊŋŌōŎŏŐőŒœŔŕŖŗŘřŚśŜŝŞşŠšŢţŤťŦŧŨũŪūŬŭŮůŰűŲųŴŵŶŷŸŹźŻżŽžſƀƁƂƃƄƅƆƇƈƉƊƋƌƍƎƏƐƑƒƓƔƕƖƗƘƙƚƛƜƝƞƟƠơƢƣƤƥƦƧƨƩƪƫƬƭƮƯưƱƲƳƴƵƶƷƸƹƺƻƼƽƾƿǀǁǂǃDŽDždžLJLjljNJNjnjǍǎǏǐǑǒǓǔǕǖǗǘǙǚǛǜǝǞǟǠǡǢǣǤǥǦǧǨǩǪǫǬǭǮǯǰDZDzdzǴǵǶǷǸǹǺǻǼǽǾǿȀȁȂȃȄȅȆȇȈȉȊȋȌȍȎȏȐȑȒȓȔȕȖȗȘșȚțȜȝȞȟȠȡȢȣȤȥȦȧȨȩȪȫȬȭȮȯȰȱȲȳȴȵȶȷȸȹȺȻȼȽȾȿɀɁɂɃɄɅɆɇɈɉɊɋɌɍɎɏͰͱͲͳʹ͵Ͷͷͺͻͼͽ;Ϳ΄΅Ά·ΈΉΊΌΎΏΐΑΒΓΔΕΖΗΘΙΚΛΜΝΞΟΠΡΣΤΥΦΧΨΩΪΫάέήίΰαβαγβδεζηθικλμνξοπρςστυφχψωϊϋόύώϏϐϑϒϓϔϕϖϗϘϙϚϛϜϝϞϟϠϡϢϣϤϥϦϧϨϩϪϫϬϭϮϯϰϱϲϳϴϵ϶ϷϸϹϺϻϼϽϾϿЀЁЂЃЄЅІЇЈЉЊЋЌЍЎЏАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюяѐёђѓєѕіїјљњћќѝўџѠѡѢѣѤѥѦѧѨѩѪѫѬѭѮѯѰѱѲѳѴѵѶѷѸѹѺѻѼѽѾѿҀҁ҂҃҄҅҆҇҈҉ҊҋҌҍҎҏҐґҒғҔҕҖҗҘҙҚқҜҝҞҟҠҡҢңҤҥҦҧҨҩҪҫҬҭҮүҰұҲҳҴҵҶҷҸҹҺһҼҽҾҿӀӁӂӃӄӅӆӇӈӉӊӋӌӍӎӏӐӑӒӓӔӕӖӗӘәӚӛӜӝӞӟӠӡӢӣӤӥӦӧӨөӪӫӬӭӮӯӰӱӲӳӴӵӶӷӸӹӺӻӼӽӾӿԀԁԂԃԄԅԆԇԈԉԊԋԌԍԎԏԐԑԒԓԔԕԖԗԘԙԚԛԜԝԞԟԠԡԢԣԤԥԱԲԳԴԵԶԷԸԹԺԻԼԽԾԿՀՁՂՃՄՅՆՇՈՉՊՋՌՍՎՏՐՑՒՓՔՕՖՙ՚՛՜՝՞՟աբգդեզէըթժիլխծկհձղճմյնշոչպջռսվտրցւփքօֆև։֊ḀḁḂḃḄḅḆḇḈḉḊḋḌḍḎḏḐḑḒḓḔḕḖḗḘḙḚḛḜḝḞḟḠḡḢḣḤḥḦḧḨḩḪḫḬḭḮḯḰḱḲḳḴḵḶḷḸḹḺḻḼḽḾḿṀṁṂṃṄṅṆṇṈṉṊṋṌṍṎṏṐṑṒṓṔṕṖṗṘṙṚṛṜṝṞṟṠṡṢṣṤṥṦṧṨṩṪṫṬṭṮṯṰṱṲṳṴṵṶṷṸṹṺṻṼṽṾṿẀẁẂẃẄẅẆẇẈẉẊẋẌẍẎẏẐẑẒẓẔẕẖẗẘẙẚẛẠạẢảẤấẦầẨẩẪẫẬậẮắẰằẲẳẴẵẶặẸẹẺẻẼẽẾếỀềỂểỄễỆệỈỉỊịỌọỎỏỐốỒồỔổỖỗỘộỚớỜờỞởỠỡỢợỤụỦủỨứỪừỬửỮữỰựỲỳỴỵỶỷỸỹ—–―—“―‘”“…‥”※…Ⅰ※ⅡⅢⅡⅣⅤⅥⅦⅧⅨⅩ←↑←→↓→−↓■−△□■▲▽▼◆▽○◆◎○●◯●★◯☆★♪☆ ♪、 。、〃。〄々〆々〇〆〈〉〈《》《「」「『」』『【』】【〒】〓〔〕〔〖〗〖〘〙〘〚〛〜〝〜〞〝〟〠〰〱〲〳〴〳〵〴〶〵〷〻〼〽ぁあぁぃあいぃぅいうぅぇうえぇぉえおぉかおがかきがぎきくぎぐくけぐげけこげごこさござさしざじしすじずすせずぜせそぜぞそたぞだたちだぢちっぢつっづつてづでてとでどとなどになぬにねぬのねはのばはぱばひぱびひぴびふぴぶふぷぶへぷべへぺべほぺぼほぽぼまぽみまむみめむもめゃもやゃゅやゆゅょゆよょらよりらるりれるろれゎろわゐわゑをんをゔんゕゖ゙゚゛゜゛ゝゞゟ゠ァアァィアイィゥイウゥェウエェォエオォカオガカキガギキクギグクケグゲケコゲゴコサゴザサシザジシスジズスセズゼセソゼゾソタゾダタチダヂチッヂツッヅツテヅデテトデドトナドニナヌニネヌノネハノバハパバヒパビヒピビフピブフプブヘプベヘペベホペボホポボマポミマムミメムモメャモヤャュヤユュョユヨョラヨリラルリレルロレヮロワヲワンヲヴンヵヴヶヵ・ヶー・一ー三万一丈丁七上三下上両且与世丙丘下不中両並主丹丸中串予久乱乗乖乞乃乳之亀乏主丼九乾了也乙事争予二事人亮交二享五亜亡互亭云井些京介仇今仄什人仁以付仙仔仏介他代仕令件仰仲以仮休伎伏伊企伐件任会休伝会何伺佇位体伸伴似佐住伝低伯例併何作余使佳便侭侘侶価侵供依侍例侮侠優偏俺俊偶個倒俎傑像係俸倹傀偲傘備僕僚僻催偉側傲信働償億傾倶借僧俗偽促儡倍傍儒値停偵候僅俯俳儘僭儀儚保俄健倫傷倣倦俵便修倉債元儲優入児党先兆充元光兄兎克兜免全入公全八内其典兵共公六兼具冒冊再内円出凱冷凸几冨写冤凍凧冥冴凶冒冗冠凛処凝凡冬凌凄凹冶凸刊刀刈切刃函分出刑刊初刑列制判到別初利刺刹券刷制前刻削剃刺則剌勘割加勉動勇効劇副力功劫剪剣勅創労劣劈勃剰劾助励剖前剛剥努剤北包匍勤務勝勘勾匕勢勿匂募化勧匐勲占博千医半十午卓協匠卑升北単匹卒南匿区却危即印占卦厚厄却卵卸原厚叔参双収厨叉及原去又厠厭反友厳古叢受叔取叡口号叱古叶叫句召叩台可史右只名司吉各同吊号合味呟呉呂向含吐吾吽周吶吼吹君名吸吠呪吟否呑呆告呈喜咽咆喚呵哀善唐員唆哨和品咳味唱咤咲喋哲命唾喉唖哺啄問哮哉商喀喘唯咄哭唄啓咀唇喊呼唸啜咎呻哄啖呷咥営喫喝喜喩喪喧喰回器噌嚥噴囃嘗嘆嘯嗟嗄嚢嗅噪嚇嘔囁噺噤嘉噂嘲噛嗤嘩営嘘囚四嘱嗚嚼嗜園団因囲回困圏固図囮国在園圧土地在圭場坐堪報堰堂垣埋地埃堕埼坦堆垢堀域坂執堤坊培城垂均埒堅基型夏壇塀墨塊塗境壱墜売塔壌士声壊填塩増壁墓塁壮墟塚塞塾場壷塵壕堵変壺夜多夕外夏夢夜大夢天大夫天太姉奉奸奔奮央妄妨妖奥夭妙夫奢好如奴妊妹奨女奈妥失奇奪套奏契妬妻娘娑姑始姉姻委姪姦姓姫姥姐姿威婦娩婚娘娶娠婆娯子媛嬉婦媒媚嫉婿嫌嫁嬢嬲嬌学季子孤孝孕存孔字害孵完実宛宙室宰客宝宇宜安宮宗官定宣守孫学宅家宴宵害察寂寛容富家寝寒宿密寄寇局尉寺寿射尚尻審将尺封寸尼寡尋察尽寵尿寥尊小寧対寮就少尾専導尖屋局屈届居屁崎層屯岡屓屍屠岩履島岐峠峡峙岳展崇屋山屑岸峰属席巫希巻巷帛崎巴州巨崖帖巡崩帆巣己嵐差嶋布巧市師左巾工帝川嶺帳帰席帯年幡幕干幣幌常帳平幅帽幇幸年店幽序幸広庁底幻幼庇幹床幾座店度府建廃庭康庫延庶廷座廊廉庸当強廻弱弧弘弟弛弔弁式張弓弾建弄弩弐弥弊引後役彦征彰待形彫彷影律彗彩彿彼当径往彙徊御従後徒得徐徘心徳微徨循徴復徹御志必忍忌心急怠怒志忽念忠怖忸思応忙忘快恋怪怨怩怯性急恨恍恐恒恋恣恥悪息悄恰恨恵悟恫悩恩悍悦悠恭悔患感惨惑愕愛悼悶悴惹悪悲愉惰惧惣惜惚愚情意悸愁想戻慌懺成懃或憩戴戚慣憊戸戮懇憑愾憫感態憔戯懐慢憎戒懸慰慟憚戦慄憬慮憶戟憤慎懣慕憾慇我慈憮慨憧愧慙憐憂懲憲慶手所扈房扉戻扇担抉披手才抑承打投抜扮払抗抱抹把抽折批扱押技托扶択抵拶拳拓括拍拙拐拠拭拌拗拮拒拝担招拡拉拘挨持拶挙挟指拾拷挑捜捗捕挺挨挿捌捉挽振挫捏推捜授掟掛探据捧掃捺捩掠掌捨採控捲掘捻排接捷撲携摂援撤揃掻措搾掴揚摺撰搦掲搬摩搭揉提損掬揮換描摯撮推揺揄撃摘播撫撒揶握撥故操擲政擁支撹撻擦攣改擬攻攫撼撲攪放擢敷故敢敬教散救敗敦敏数敵整文敷斃断斎斐斡文料斜斗斬斥斧斉新断既施新方於旗族旋旅日既明旬昆昂昇旧日旦早旨旺昼昭昏昧映是易明星春昔昨暇晦晒普晩晶昼晴暁智晋景時暗暈暖暑暇書曜曝曲暦曇暮曖更暴暢暗曰服書有曽替曾月曹朋最望服朗末朝朦望期未木朧本末条李朱杞札束村机本杏杓杖杜杉朶朴材朽来条東来杯板東杵杳松林板枕析染柄林架枝柑某果枷枯枠枚枢査柔柳柱柩柵柴染格核栗根査校株栄柿栓検棘椎梗棹桔桂棒椀梳森梯棄械棲案桁棋椅桃梅棺棚梱植棟桶桜梁栽梶格梨梢業楕椿楊検楚楔標槍楯極槌楼概榴榊槽業構様榜楽樋歯歓標正橋模樽欒樹止権歪檀橙武歌橘欺樺檻欲櫛欄欠櫃櫓檄次機欝歩欧横此死歳歴歯殺殲死殊段殴殆殖残殉母殻殺殿毅毀気毯氈母比民毎毛氏毟毒毫求永氷氾気汁水沙沖沈沌池汲求汎汝汚汰汗江沁決汽沢没沙派沢泥沼洗泡津沿況泰洞泉油注波沽沫河泳洩洋法洒活泄洪泣沸泊泌治流派深浪淋浮涼浜流淫淡淀浄涸涜淘涌浸消液浅海浦浴淑浣涎涯涙混淵深港済渇淹渉添減温渡渋測混清渦渚渓渠準湾溌港湿湯源湖湛満湧渾滅溜溶溺準溢溝潰滴漠漿演潔潜漏滞漲滑漆漫漑滲漕漂潤滔漬滅漢滝漁潮火澪濃潰瀕澄澹澤瀟濛激瀉瀬澱濡濯濫瀑濁灌澳災火灰灸灯灼無炭点烈焙焔烙炊烹為炙焚災炒炎烏炸焉炉照煌然焦無焼煤煙煥煎父爆煽熱煩爵燻燥熊燈爪燭照熟燗熾燃燦煮熨爛版爽片爺父牲牧版牒牢牙牌物牛特牲犠特牽犯犬犠獄狂狽狡犯猟狙猿狗猥独猶猪狐狛狼献猛猜状狸猾猫狩狭現玄玉玩獅獪獄獣珍班珀王率珈獰珠獲生環甘琲琴瑞瓢璧琵瑣瓜瓦甚琶現球瑕瓶瑾琥理用生甥産甦由田用画申町甲由男疲疎畠疑畜疫畔畑畳異疇界畦畏略番留画病疼疲疾疹疽療痺病瘴痩痛痔痕痴痘痍痒痣痙瘍症発癇療癪癒癖盥発皇監皮皴盗皆登白盤的盆皺盟皿百益盛目盧盪盥省相目盲盾直石瞞真矜眷看眼眺眩睨瞳矮矯睦督眉瞬矛睡眠知瞭着矢瞥瞑短睫瞼睛眈県省研砂石破砦砥砕研砲礁硫碑硬碗硝磨確磔磐磯碁破磁示礫礎礁社示礼神祉祀祓社祈祐祖祝祟神祭祟祠祥票科禽福禊禍禄秀禎私禮祷秋祭禁稿稀稜稲称税移科稽秤稼秒秘秦稚秩種程積穂稿穀究穢穫積穴穏空究第端窪笑競窮笠突竹竦窄章窺竿穿空窒窓笛符童竜立窃窟窩箱筋箇筑等答筈第筆箔算筒管策箒箋箪粉簀築篭範簿簡箱米篠節籠箸籍簒篝篤紅粧糸粋糊粉精粥粟系粘糧粗粛糖粒約紀糾糞紙紐純紋納紅紗級紙紹級紡累素紫紳細索紛終紹紺組終統経絆絢組結給絨絡絞絵統綿綜綻絵絹綴綱網綺継続絶維綬緊綿線総緑緊緒緘緋編線緞締罪繕緯缶織繊緩繰縋繋縦縮練縛繍繁罠編縷縁縫績纏縞縄緻署罪置罰老翡義羊翻翌罷群習署美翠翳翔翁羞羅罹罵羹羽翼羨考老者考聞耽耗聖者耕耐聘耳脅胆胡胞胃聡肴能肛聞肪胴胸肖聳股職肢肝脂肘肯育肉聴背肋肥肺肌胎肩脚脊脈脅脆脇自腋膨腐膠臍膚腫脚膝腱臨腔腰腑腸膏腕臆腺脛腹膳脳膜膿腿脱臣臓臼至致自臭茜般航色舵茅芻艱苛花芸舅興芝舗艦船芽苺舌苗若茂芯臼芋茎芳舟英苔舎苦舐艶良芥艇舞茶茫茜茨荘草荒茹茸茶落華萱萬荘莫菊荷菌萌莢菓菩萎菅菜蒐葦落著葵葱葉葛葺董葬蜜蔓蓮蛆薩薄蚊藁藻蒼蒔藩蜃虎藺薫薪蛙蒙蕉蘇蛇藍蛛蜂虹虚蛍蚕虐蒐蔽蔑蒸蓄蛸蛮薙薬薔薦蓋蒲藉薮蕎藤蓑蜘蛾薇藪虜蔵藹蘭虔虫表衆衛蝉蜜蟹蝶蝕蝿街蟷血衡蝸衣蟻術螺衝蝋行融螂蠢要褐表裏襲裔襟袋袖複衷袂褻裟裸西裁裂裾襖袈装製褄褒褪袴衰被裕補見要覇覆規見覧規覚視覗親覧言親解角観触計訃訂言記訓討計託訊設訪訣訛訟記訝訥訳許設診訴訳詩詠詐証詣評詈詞診試詛話詭詩詰詫詮誌該話誅詳誇誉誘誌認誓誑誕語誘読説誤誠誦語課読誰調誼課誹請調談講諦謙諾請諺論謀謎諜諭謂諍諤謗諌諮諸賓豆護識貼資財讒貌貯貧譫貢賊謹謳貝貴貰貪讐講貶賀谷買賃貫警謝謡責豚豊貸貞賑費貿負譲豪賂議象賎販譜貨賄讃豹路趣跋距賢賭走賛贄跨賠賤趨赴赤越跡購質賜賦贅超跪贈足賽賓賞起贔赦贖車蹲践躊蹄躁躯躇身躍蹂躙踏路跳踵踪躾躓蹴踊転軌軍軟車軒軋迫辞迎辺輩迦辱軸轄農較辜輿軽迅辿辣近込迂辰返辛転轍辻載輪輸輝辟轢轟通透逗送這途迭迸逃述逐退逅逆追迫迷週逝逞逢速連逡通逮造進週達遊遍遅過運進遂逼道逸遇遁選遵遙適違遭達遣遥遮遷遡遠遜遺選部那邪邸還郎邊邁邏遽邂郊邦遺避都郵部郷郭重釈酢醸里酬醜醇配醤酌酷醍醐酩都酊酔采酒酎酸鄙醒鈴釣金鈍重針野量釘釜録鉈銃鉤鉱鋏錐錆鋸鉢錠銛鋒鈴錦鋼銘銭錯鉄鋲銀鋭錬鉛銅長鑽鏡鍵鎧鍛鍬鍋鑑鎌録鍔鐘鎖鎮閉長閂閃門間閑開閉際阿除隙陶限閨関隕間階闇陀陣闢隘隊陰閥降隆隔隈閻陸陳阻陽阜院閣闊阪随険隅陛闘闖閲陥防隠障際雀隻隣隷隠集雁雀雅雄雛雉雌雑雇集雨離雛難雪雨雲雫雰雪電雲零雷非需電露震霊静霞霧青霜面非音面韋鞘革鞍鞭靭鞄鞋鞠靴順韻響音頃項頂頭預順頬頒頓須頚頑領顛頷顔額頸顎頼頭顕頻願題鬼顧餉飾饐館飛騎養騨餅高餓駒馴香馳骸飽飢風饉餞髭駆馬髪類騒餡飴騰驚骨飯飲鬱験駕駄飄髄顛騙餐首食颯駅餌饅駐飼饒驕魅魁魄鬼魂魎魍魅魔魎魑麦麓鷲鷺鳥鴉鳩鶏鮭鳶鷹鯉鶴魚鹸鳳鵜鱗麗鮮鴨鯨鶯鹿鱈鮫鳴魔鵠鯛機!齢鼓龍鼎齎黄齬黒麦麺黙齟鼻麿鼠麻"!#$#%$&%'&()(*)+*,+-,.-/.0/102132435465768798:9;:<=<>=?>@?A@BACBDCEDFEGFHGIHJIKJLKMLNMONPOQPRQSRTSUTVUWVXWYXZY[Z\]^_`_abacbdcedfegfhgihjiklkmlnmonpoqprsrtsutvuwxwyxzy{|{}~⦅~⦆⦅。「」、・ヲァィゥェォャュョッーアイウエオカキクケコサシスセソタチツテトナニヌネノハヒフヘホマミムメモヤユヨラリルレロワン゙゚¥
Expand Down

0 comments on commit 3e7f7aa

Please sign in to comment.