sum_counts.py (949B)
1 import operator 2 import timeit 3 from collections import Counter 4 from pathlib import Path 5 6 7 LANG = 'hr' 8 9 10 def read_text_counts(filepath: Path) -> Counter: 11 c = Counter() 12 13 with open(filepath, 'rt') as f: 14 for line in f: 15 word, count, _percentage = line.strip().split('\t') 16 c[word] += int(count) 17 18 return c 19 20 21 def main(): 22 files = list(Path(f'wiki_count/{LANG}wiki/').glob('*')) 23 total = len(files) 24 25 c = Counter() 26 27 last_time = timeit.default_timer() 28 for i, file in enumerate(files): 29 if i % 1000 == 0: 30 now = timeit.default_timer() 31 print(f'{i} / {total} ({now - last_time})') 32 last_time = now 33 34 c.update(read_text_counts(file)) 35 36 with open(f'data/{LANG}wiki/all', 'wt') as f: 37 for word, count in sorted(c.items(), key=operator.itemgetter(1), reverse=True): 38 f.write(f'{word}\t{count}\n') 39 40 41 if __name__ == '__main__': 42 main()