|
|
|
|
|
import sys |
|
import os |
|
import re |
|
import argparse |
|
import subprocess |
|
import shutil |
|
from threading import Timer |
|
from datetime import datetime |
|
|
|
|
|
def run_cmd(cmd, timeout_sec=30): |
|
proc = subprocess.Popen(cmd, shell=True) |
|
kill_proc = lambda p: p.kill() |
|
timer = Timer(timeout_sec, kill_proc, [proc]) |
|
try: |
|
timer.start() |
|
stdout,stderr = proc.communicate() |
|
finally: |
|
timer.cancel() |
|
|
|
def tokenize_latex(latex_code, latex_type="", middle_file=""): |
|
if not latex_code: |
|
return False, latex_code |
|
if not latex_type: |
|
latex_type = "tabular" if "tabular" in latex_code else "formula" |
|
if not middle_file: |
|
middle_file = "out-" + datetime.now().strftime('%Y_%m_%d_%H_%M_%S') + ".txt" |
|
temp_file = middle_file + '.tmp' |
|
|
|
if latex_type == "formula": |
|
with open(temp_file, 'w') as f: |
|
prepre = latex_code |
|
|
|
prepre = re.sub(r'\\begin{(split|align|alignedat|alignat|eqnarray)\*?}(.+?)\\end{\1\*?}', r'\\begin{aligned}\2\\end{aligned}', prepre, flags=re.S) |
|
prepre = re.sub(r'\\begin{(smallmatrix)\*?}(.+?)\\end{\1\*?}', r'\\begin{matrix}\2\\end{matrix}', prepre, flags=re.S) |
|
f.write(prepre) |
|
|
|
cmd = r"cat %s | node %s %s > %s " % (temp_file, os.path.join(os.path.dirname(__file__), 'preprocess_formula.js'), 'normalize', middle_file) |
|
ret = subprocess.call(cmd, shell=True) |
|
os.remove(temp_file) |
|
if ret != 0: |
|
return False, latex_code |
|
|
|
operators = '\s?'.join('|'.join(['arccos', 'arcsin', 'arctan', 'arg', 'cos', 'cosh', 'cot', 'coth', 'csc', 'deg', 'det', 'dim', 'exp', 'gcd', 'hom', 'inf', |
|
'injlim', 'ker', 'lg', 'lim', 'liminf', 'limsup', 'ln', 'log', 'max', 'min', 'Pr', 'projlim', 'sec', 'sin', 'sinh', 'sup', 'tan', 'tanh'])) |
|
ops = re.compile(r'\\operatorname {(%s)}' % operators) |
|
with open(middle_file, 'r') as fin: |
|
for line in fin: |
|
tokens = line.strip().split() |
|
tokens_out = [] |
|
for token in tokens: |
|
tokens_out.append(token) |
|
post = ' '.join(tokens_out) |
|
|
|
names = ['\\'+x.replace(' ', '') for x in re.findall(ops, post)] |
|
post = re.sub(ops, lambda match: str(names.pop(0)), post).replace(r'\\ \end{array}', r'\end{array}') |
|
os.remove(middle_file) |
|
return True, post |
|
|
|
elif latex_type == "tabular": |
|
latex_code = latex_code.replace("\\\\%", "\\\\ %") |
|
latex_code = latex_code.replace("\%", "<PERCENTAGE_TOKEN>") |
|
latex_code = latex_code.split('%')[0] |
|
latex_code = latex_code.replace("<PERCENTAGE_TOKEN>", "\%") |
|
if not "\\end{tabular}" in latex_code: |
|
latex_code += "\\end{tabular}" |
|
with open(middle_file, 'w') as f: |
|
f.write(latex_code.replace('\r', ' ').replace('\n', ' ')) |
|
cmd = "perl -pe 's|hskip(.*?)(cm\\|in\\|pt\\|mm\\|em)|hspace{\\1\\2}|g' %s > %s"%(middle_file, temp_file) |
|
ret = subprocess.call(cmd, shell=True) |
|
if ret != 0: |
|
return False, latex_code |
|
os.remove(middle_file) |
|
cmd = r"cat %s | node %s %s > %s " % (temp_file, os.path.join(os.path.dirname(__file__), 'preprocess_tabular.js'), 'tokenize', middle_file) |
|
ret = subprocess.call(cmd, shell=True) |
|
os.remove(temp_file) |
|
if ret != 0: |
|
return False, latex_code |
|
with open(middle_file, 'r') as fin: |
|
for line in fin: |
|
tokens = line.strip().split() |
|
tokens_out = [] |
|
for token in tokens: |
|
tokens_out.append(token) |
|
post = ' '.join(tokens_out) |
|
os.remove(middle_file) |
|
return True, post |
|
else: |
|
print(f"latex type{latex_type} unrecognized.") |
|
return False, latex_code |
|
|
|
if __name__ == '__main__': |
|
latex_code = open("2.txt", 'r').read().replace('\r', ' ') |
|
print("=>", latex_code) |
|
new_code = tokenize_latex(latex_code) |
|
print("=>", new_code) |
